From 72f87bfa381ed7a5b74fb5907f932b78d0257ab9 Mon Sep 17 00:00:00 2001 From: mihir-4116 Date: Fri, 8 Sep 2023 11:26:24 +0530 Subject: [PATCH 1/5] feat(tiktok_ads): messageId to event_id mapping support --- src/v0/destinations/tiktok_ads/data/TikTokTrack.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v0/destinations/tiktok_ads/data/TikTokTrack.json b/src/v0/destinations/tiktok_ads/data/TikTokTrack.json index 05709af27a..b15223b999 100644 --- a/src/v0/destinations/tiktok_ads/data/TikTokTrack.json +++ b/src/v0/destinations/tiktok_ads/data/TikTokTrack.json @@ -1,7 +1,7 @@ [ { "destKey": "event_id", - "sourceKeys": "properties.eventId", + "sourceKeys": ["properties.eventId", "properties.event_id", "messageId"], "required": false }, { From e45536805cf9545b73f4d5bf1be5fad1565ab075 Mon Sep 17 00:00:00 2001 From: Akash Chetty Date: Fri, 8 Sep 2023 20:28:29 +0530 Subject: [PATCH 2/5] fix: json paths for non tracks events for warehouse (#2571) --- src/warehouse/index.js | 114 ++- src/warehouse/util.js | 64 +- .../integrations/jsonpaths/legacy/aliases.js | 427 +++++++++ .../integrations/jsonpaths/legacy/extract.js | 278 ++++++ .../integrations/jsonpaths/legacy/groups.js | 432 +++++++++ .../jsonpaths/legacy/identifies.js | 850 ++++++++++++++++++ .../integrations/jsonpaths/legacy/pages.js | 405 +++++++++ .../integrations/jsonpaths/legacy/screens.js | 405 +++++++++ .../integrations/jsonpaths/legacy/tracks.js | 830 +++++++++++++++++ .../integrations/jsonpaths/new/aliases.js | 415 +++++++++ .../integrations/jsonpaths/new/extract.js | 278 ++++++ .../integrations/jsonpaths/new/groups.js | 420 +++++++++ .../integrations/jsonpaths/new/identifies.js | 850 ++++++++++++++++++ .../integrations/jsonpaths/new/pages.js | 393 ++++++++ .../integrations/jsonpaths/new/screens.js | 393 ++++++++ .../integrations/jsonpaths/new/tracks.js | 830 +++++++++++++++++ test/__tests__/flatten_events.test.js | 44 +- test/__tests__/warehouse.test.js | 71 +- 18 files changed, 7426 insertions(+), 73 deletions(-) create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js create mode 100644 test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js diff --git a/src/warehouse/index.js b/src/warehouse/index.js index dd5116f00e..244605613f 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -7,7 +7,8 @@ const { isObject, isBlank, isValidJsonPathKey, - getKeysFromJsonPaths, + isValidLegacyJsonPathKey, + keysFromJsonPaths, validTimestamp, getVersionedUtils, isRudderSourcesEvent, @@ -195,45 +196,51 @@ function setDataFromColumnMappingAndComputeColumnTypes( columnTypes = {context_library_name: 'string', context_library_version: 'string'} */ - function setDataFromInputAndComputeColumnTypes( - utils, - eventType, - output, - input, - columnTypes, - options, - prefix = '', - level = 0, + utils, + eventType, + output, + input, + columnTypes, + options, + completePrefix = '', + completeLevel = 0, + prefix = '', + level = 0, ) { if (!input || !isObject(input)) return; Object.keys(input).forEach((key) => { - if (isValidJsonPathKey(eventType, `${prefix + key}`, input[key], level, options.jsonKeys)) { + const isValidLegacyJSONPath = isValidLegacyJsonPathKey(eventType, `${prefix + key}`, level, options.jsonLegacyPathKeys); + const isValidJSONPath = isValidJsonPathKey(`${completePrefix + key}`, completeLevel, options.jsonPathKeys); + + if (isValidJSONPath || isValidLegacyJSONPath) { if (isBlank(input[key])) { return; } const val = JSON.stringify(input[key]); appendColumnNameAndType( - utils, - eventType, - `${prefix + key}`, - val, - output, - columnTypes, - options, - true, + utils, + eventType, + `${prefix + key}`, + val, + output, + columnTypes, + options, + true, ); } else if (isObject(input[key]) && (options.sourceCategory !== 'cloud' || level < 3)) { setDataFromInputAndComputeColumnTypes( - utils, - eventType, - output, - input[key], - columnTypes, - options, - `${prefix + key}_`, - level + 1, + utils, + eventType, + output, + input[key], + columnTypes, + options, + `${completePrefix + key}_`, + completeLevel + 1, + `${prefix + key}_`, + level + 1, ); } else { let val = input[key]; @@ -245,13 +252,13 @@ function setDataFromInputAndComputeColumnTypes( val = JSON.stringify(val); } appendColumnNameAndType( - utils, - eventType, - `${prefix + key}`, - val, - output, - columnTypes, - options, + utils, + eventType, + `${prefix + key}`, + val, + output, + columnTypes, + options, ); } }); @@ -316,12 +323,15 @@ function storeRudderEvent(utils, message, output, columnTypes, options) { function addJsonKeysToOptions(options) { // Add json key paths from integration options and destination config const jsonPaths = Array.isArray(options.integrationOptions?.jsonPaths) - ? options.integrationOptions.jsonPaths - : []; + ? options.integrationOptions.jsonPaths + : []; if (options.destJsonPaths) { jsonPaths.push(...options.destJsonPaths.split(',')); } - options.jsonKeys = getKeysFromJsonPaths(jsonPaths); + + const keys = keysFromJsonPaths(jsonPaths); + options.jsonPathKeys = keys.jsonPathKeys; + options.jsonLegacyPathKeys = keys.jsonLegacyPathKeys; } /* @@ -583,6 +593,8 @@ function processWarehouseMessage(message, options) { message.context, commonColumnTypes, options, + `${eventType + '_context_'}`, + 2, 'context_', ); @@ -605,6 +617,8 @@ function processWarehouseMessage(message, options) { message.properties, eventTableColumnTypes, options, + `${eventType + '_properties_'}`, + 2, ); setDataFromColumnMappingAndComputeColumnTypes( utils, @@ -658,6 +672,8 @@ function processWarehouseMessage(message, options) { message.context, commonColumnTypes, options, + `${eventType + '_context_'}`, + 2, 'context_', ); setDataFromColumnMappingAndComputeColumnTypes( @@ -732,6 +748,8 @@ function processWarehouseMessage(message, options) { message.properties, eventTableColumnTypes, options, + `${eventType + '_properties_'}`, + 2, ); setDataFromInputAndComputeColumnTypes( utils, @@ -740,6 +758,8 @@ function processWarehouseMessage(message, options) { message.userProperties, eventTableColumnTypes, options, + `${eventType + '_userProperties_'}`, + 2, ); setDataFromColumnMappingAndComputeColumnTypes( utils, @@ -794,6 +814,8 @@ function processWarehouseMessage(message, options) { message.userProperties, commonColumnTypes, options, + `${eventType + '_userProperties_'}`, + 2, ); setDataFromInputAndComputeColumnTypes( utils, @@ -802,6 +824,8 @@ function processWarehouseMessage(message, options) { message.context ? message.context.traits : {}, commonColumnTypes, options, + `${eventType + '_context_traits_'}`, + 3, ); setDataFromInputAndComputeColumnTypes( utils, @@ -810,6 +834,8 @@ function processWarehouseMessage(message, options) { message.traits, commonColumnTypes, options, + `${eventType + '_traits_'}`, + 2, '', ); @@ -821,6 +847,8 @@ function processWarehouseMessage(message, options) { message.context, commonColumnTypes, options, + `${eventType + '_context_'}`, + 2, 'context_', ); @@ -925,6 +953,8 @@ function processWarehouseMessage(message, options) { message.properties, columnTypes, options, + `${eventType + '_properties_'}`, + 2, ); // set rudder properties after user set properties to prevent overwriting setDataFromInputAndComputeColumnTypes( @@ -934,6 +964,8 @@ function processWarehouseMessage(message, options) { message.context, columnTypes, options, + `${eventType + '_context_'}`, + 2, 'context_', ); setDataFromColumnMappingAndComputeColumnTypes( @@ -992,6 +1024,8 @@ function processWarehouseMessage(message, options) { message.traits, columnTypes, options, + `${eventType + '_traits_'}`, + 2, ); setDataFromInputAndComputeColumnTypes( utils, @@ -1000,6 +1034,8 @@ function processWarehouseMessage(message, options) { message.context, columnTypes, options, + `${eventType + '_context_'}`, + 2, 'context_', ); setDataFromColumnMappingAndComputeColumnTypes( @@ -1046,6 +1082,8 @@ function processWarehouseMessage(message, options) { message.traits, columnTypes, options, + `${eventType + '_traits_'}`, + 2, ); setDataFromInputAndComputeColumnTypes( utils, @@ -1054,6 +1092,8 @@ function processWarehouseMessage(message, options) { message.context, columnTypes, options, + `${eventType + '_context_'}`, + 2, 'context_', ); setDataFromColumnMappingAndComputeColumnTypes( diff --git a/src/warehouse/util.js b/src/warehouse/util.js index b965ecb029..db81d11cbd 100644 --- a/src/warehouse/util.js +++ b/src/warehouse/util.js @@ -15,26 +15,69 @@ const isObject = (value) => { return value != null && (type === 'object' || type === 'function') && !Array.isArray(value); }; -const isValidJsonPathKey = (eventType, key, val, level, jsonKeys = {}) => { +const isValidJsonPathKey = (key, level, jsonKeys = {}) => { + return jsonKeys[key] === level; +}; +const isValidLegacyJsonPathKey = (eventType, key, level, jsonKeys = {}) => { return eventType === 'track' && jsonKeys[key] === level; }; const isBlank = (value) => { return _.isEmpty(_.toString(value)); }; + /* - * input => ["a", "b.c"] - * output => { "a": 0, "b_c": 1} +This function takes in an array of json paths and returns an object with keys as the json path and value as the position of the key in the json path +Example: +Input: +[ + "a", + "b.c" +] +Output: +{ + "a": 0, + "b_c": 1 +} + +Input: +[ + "track.context.a", + "track.properties.b", + "pages.properties.c.d", + "groups.traits.e.f" +] +Output: +{ + "track_context_a": 2, + "track_properties_b": 2, + "pages_properties_c_d": 3, + "groups_traits_e_f": 3 +} */ -const getKeysFromJsonPaths = (jsonPaths) => { - const jsonKeys = {}; +const keysFromJsonPaths = (jsonPaths) => { + const jsonPathKeys = {}; + const jsonLegacyPathKeys = {}; + + const supportedEventPrefixes = ['track.', 'identify.', 'page.', 'screen.', 'alias.', 'group.', 'extract.']; + jsonPaths.forEach((jsonPath) => { - if (jsonPath.trim()) { - const paths = jsonPath.trim().split('.'); - jsonKeys[paths.join('_')] = paths.length - 1; + const trimmedJSONPath = jsonPath.trim(); + if (!trimmedJSONPath) { + return; + } + + const paths = trimmedJSONPath.split('.'); + const key = paths.join('_'); + const pos = paths.length - 1; + + if (supportedEventPrefixes.some(prefix => trimmedJSONPath.startsWith(prefix))) { + jsonPathKeys[key] = pos; + return; } + jsonLegacyPathKeys[key] = pos; }); - return jsonKeys; + return {jsonPathKeys, jsonLegacyPathKeys}; }; // https://www.myintervals.com/blog/2009/05/20/iso-8601-date-validation-that-doesnt-suck/ @@ -98,7 +141,8 @@ module.exports = { isObject, isBlank, isValidJsonPathKey, - getKeysFromJsonPaths, + isValidLegacyJsonPathKey, + keysFromJsonPaths, timestampRegex, validTimestamp, getVersionedUtils, diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js new file mode 100644 index 0000000000..ee748a1a2b --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/aliases.js @@ -0,0 +1,427 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey", + testMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + ctestMap: { + cnestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-GB", + screen: { + density: 2 + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "alias", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "aliases" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "aliases" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "aliases" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "aliases" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "ALIASES" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js new file mode 100644 index 0000000000..906c3ada23 --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/extract.js @@ -0,0 +1,278 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + sources: { + job_id: "djfhksdjhfkjdhfkjahkf", + version: "1169/merge", + job_run_id: "job_run_id", + task_run_id: "task_run_id" + }, + CMap: { + nestedMap: { + n1: "context nested prop 1" + } + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + event: "Product Added", + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + recordId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { + currency: "USD", + revenue: 50, + PMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + type: "extract", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_sources_job_id": "djfhksdjhfkjdhfkjahkf", + "context_sources_job_run_id": "job_run_id", + "context_sources_task_run_id": "task_run_id", + "context_sources_version": "1169/merge", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "event": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "p_map_nested_map_n_1": "nested prop 1", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50 + }, + "metadata": { + "columns": { + "context_c_map_nested_map_n_1": "string", + "context_sources_job_id": "string", + "context_sources_job_run_id": "string", + "context_sources_task_run_id": "string", + "context_sources_version": "string", + "context_user_agent": "string", + "currency": "string", + "event": "string", + "id": "string", + "p_map_nested_map_n_1": "string", + "received_at": "datetime", + "revenue": "int", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + rs: [ + { + "data": { + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_sources_job_id": "djfhksdjhfkjdhfkjahkf", + "context_sources_job_run_id": "job_run_id", + "context_sources_task_run_id": "task_run_id", + "context_sources_version": "1169/merge", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "event": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "p_map_nested_map_n_1": "nested prop 1", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50 + }, + "metadata": { + "columns": { + "context_c_map_nested_map_n_1": "string", + "context_sources_job_id": "string", + "context_sources_job_run_id": "string", + "context_sources_task_run_id": "string", + "context_sources_version": "string", + "context_user_agent": "string", + "currency": "string", + "event": "string", + "id": "string", + "p_map_nested_map_n_1": "string", + "received_at": "datetime", + "revenue": "int", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + bq: [ + { + "data": { + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_sources_job_id": "djfhksdjhfkjdhfkjahkf", + "context_sources_job_run_id": "job_run_id", + "context_sources_task_run_id": "task_run_id", + "context_sources_version": "1169/merge", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "event": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "p_map_nested_map_n_1": "nested prop 1", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50 + }, + "metadata": { + "columns": { + "context_c_map_nested_map_n_1": "string", + "context_sources_job_id": "string", + "context_sources_job_run_id": "string", + "context_sources_task_run_id": "string", + "context_sources_version": "string", + "context_user_agent": "string", + "currency": "string", + "event": "string", + "id": "string", + "loaded_at": "datetime", + "p_map_nested_map_n_1": "string", + "received_at": "datetime", + "revenue": "int", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + postgres: [ + { + "data": { + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_sources_job_id": "djfhksdjhfkjdhfkjahkf", + "context_sources_job_run_id": "job_run_id", + "context_sources_task_run_id": "task_run_id", + "context_sources_version": "1169/merge", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "event": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "p_map_nested_map_n_1": "nested prop 1", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50 + }, + "metadata": { + "columns": { + "context_c_map_nested_map_n_1": "string", + "context_sources_job_id": "string", + "context_sources_job_run_id": "string", + "context_sources_task_run_id": "string", + "context_sources_version": "string", + "context_user_agent": "string", + "currency": "string", + "event": "string", + "id": "string", + "p_map_nested_map_n_1": "string", + "received_at": "datetime", + "revenue": "int", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + snowflake: [ + { + "data": { + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_SOURCES_JOB_ID": "djfhksdjhfkjdhfkjahkf", + "CONTEXT_SOURCES_JOB_RUN_ID": "job_run_id", + "CONTEXT_SOURCES_TASK_RUN_ID": "task_run_id", + "CONTEXT_SOURCES_VERSION": "1169/merge", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EVENT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "P_MAP_NESTED_MAP_N_1": "nested prop 1", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50 + }, + "metadata": { + "columns": { + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_SOURCES_JOB_ID": "string", + "CONTEXT_SOURCES_JOB_RUN_ID": "string", + "CONTEXT_SOURCES_TASK_RUN_ID": "string", + "CONTEXT_SOURCES_VERSION": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EVENT": "string", + "ID": "string", + "P_MAP_NESTED_MAP_N_1": "string", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } + ] + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js new file mode 100644 index 0000000000..13b243f3a7 --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/groups.js @@ -0,0 +1,432 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey", + testMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + ctestMap: { + cnestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-GB", + screen: { + density: 2 + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + GCS_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "group", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "groups" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "groups" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "_groups" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "groups" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "GROUPS" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js new file mode 100644 index 0000000000..47b7f1209a --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/identifies.js @@ -0,0 +1,850 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + type: "identify", + sentAt: "2021-01-03T17:02:53.195Z", + userId: "user123", + channel: "web", + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "CTMap.nestedMap", + "TMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "CTMap.nestedMap", + "TMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "CTMap.nestedMap", + "TMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "CTMap.nestedMap", + "TMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + userProperties: { + email: "test@gmail.com", + UPMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + traits: { + TMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + context: { + os: { + "name": "android", + "version": "1.12.3" + }, + app: { + name: "RudderLabs JavaScript SDK", + build: "1.0.0", + version: "1.1.11", + namespace: "com.rudderlabs.javascript" + }, + traits: { + email: "user123@email.com", + phone: "+917836362334", + userId: "user123", + CTMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + CMap: { + nestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-US", + device: { + token: "token", + id: "id", + type: "ios" + }, + library: { + name: "RudderLabs JavaScript SDK", + version: "1.1.11" + }, + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0" + }, + rudderId: "8f8fa6b5-8e24-489c-8e22-61f23f2e364f", + messageId: "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + anonymousId: "97c46c81-3140-456d-b2a9-690d70aaca35", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + timestamp: "2020-01-24T11:59:02.403+05:30" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "loaded_at": "datetime", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP_N_1": "nested prop 1", + "UP_MAP_NESTED_MAP_N_1": "nested prop 1", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP_N_1": "string", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "T_MAP_NESTED_MAP_N_1": "string", + "UP_MAP_NESTED_MAP_N_1": "string", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + { + "data": { + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP_N_1": "nested prop 1", + "EMAIL": "user123@email.com", + "ID": "user123", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP_N_1": "nested prop 1", + "UP_MAP_NESTED_MAP_N_1": "nested prop 1" + }, + "metadata": { + "columns": { + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP_N_1": "string", + "EMAIL": "string", + "ID": "string", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "T_MAP_NESTED_MAP_N_1": "string", + "UP_MAP_NESTED_MAP_N_1": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "USERS" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js new file mode 100644 index 0000000000..0aac8a3c23 --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/pages.js @@ -0,0 +1,405 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + ctestMap: { + cnestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-GB", + screen: { + density: 2 + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "testMap.nestedMap", + "ctestMap.cnestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { + currency: "USD", + revenue: 50, + testMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "page", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "pages" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "pages" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "pages" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "pages" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PAGES" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js new file mode 100644 index 0000000000..bad325c908 --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/screens.js @@ -0,0 +1,405 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + ctestMap: { + cnestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-GB", + screen: { + density: 2 + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "testMap.nestedMap", + ".ctestMap.cnestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "testMap.nestedMap", + ".ctestMap.cnestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "testMap.nestedMap", + ".ctestMap.cnestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "testMap.nestedMap", + ".ctestMap.cnestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { + currency: "USD", + revenue: 50, + testMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "screen", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "screens" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "screens" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "screens" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "screens" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP_N_1": "nested prop 1", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP_N_1": "string", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "SCREENS" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js new file mode 100644 index 0000000000..5070284e99 --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/legacy/tracks.js @@ -0,0 +1,830 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + screen: { + density: 2 + }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + CMap: { + nestedMap: { + n1: "context nested prop 1" + } + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + event: "Product Added", + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "UPMap.nestedMap", + "PMap.nestedMap", + "CMap.nestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + userProperties: { + email: "test@gmail.com", + UPMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + properties: { + currency: "USD", + revenue: 50, + PMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "track", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "tracks" + } + }, + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "email": "test@gmail.com", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "p_map_nested_map_n_1": "nested prop 1", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "currency": "string", + "email": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "p_map_nested_map_n_1": "string", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "tracks" + } + }, + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "email": "test@gmail.com", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "currency": "string", + "email": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "p_map_nested_map": "json", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "timestamp": "datetime", + "up_map_nested_map": "json", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "event": "string", + "event_text": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "tracks" + } + }, + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "email": "test@gmail.com", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "currency": "string", + "email": "string", + "event": "string", + "event_text": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "p_map_nested_map": "string", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "timestamp": "datetime", + "up_map_nested_map": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "tracks" + } + }, + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "email": "test@gmail.com", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "currency": "string", + "email": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "p_map_nested_map": "json", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "timestamp": "datetime", + "up_map_nested_map": "json", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "TRACKS" + } + }, + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "context nested prop 1", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EMAIL": "test@gmail.com", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP_N_1": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EMAIL": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } + ] + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js new file mode 100644 index 0000000000..91f19c5c77 --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/aliases.js @@ -0,0 +1,415 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey", + testMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + ctestMap: { + cnestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-GB", + screen: { + density: 2 + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true, + RS: { + options: { + jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] + } + }, + BQ: { + options: { + jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] + } + }, + POSTGRES: { + options: { + jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: ["alias.traits.testMap.nestedMap", "alias.context.ctestMap.cnestedMap"] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "alias", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "aliases" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map": "json", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "aliases" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "aliases" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map": "json", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "aliases" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "ALIASES" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js new file mode 100644 index 0000000000..7a36e4787e --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/extract.js @@ -0,0 +1,278 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + sources: { + job_id: "djfhksdjhfkjdhfkjahkf", + version: "1169/merge", + job_run_id: "job_run_id", + task_run_id: "task_run_id" + }, + CMap: { + nestedMap: { + n1: "context nested prop 1" + } + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + event: "Product Added", + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "extract.properties.PMap.nestedMap", + "extract.context.CMap.nestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "extract.properties.PMap.nestedMap", + "extract.context.CMap.nestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "extract.properties.PMap.nestedMap", + "extract.context.CMap.nestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "extract.properties.PMap.nestedMap", + "extract.context.CMap.nestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + recordId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { + currency: "USD", + revenue: 50, + PMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + type: "extract", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_sources_job_id": "djfhksdjhfkjdhfkjahkf", + "context_sources_job_run_id": "job_run_id", + "context_sources_task_run_id": "task_run_id", + "context_sources_version": "1169/merge", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "event": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "p_map_nested_map_n_1": "nested prop 1", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50 + }, + "metadata": { + "columns": { + "context_c_map_nested_map_n_1": "string", + "context_sources_job_id": "string", + "context_sources_job_run_id": "string", + "context_sources_task_run_id": "string", + "context_sources_version": "string", + "context_user_agent": "string", + "currency": "string", + "event": "string", + "id": "string", + "p_map_nested_map_n_1": "string", + "received_at": "datetime", + "revenue": "int", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + rs: [ + { + "data": { + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_sources_job_id": "djfhksdjhfkjdhfkjahkf", + "context_sources_job_run_id": "job_run_id", + "context_sources_task_run_id": "task_run_id", + "context_sources_version": "1169/merge", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "event": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50 + }, + "metadata": { + "columns": { + "context_c_map_nested_map": "json", + "context_sources_job_id": "string", + "context_sources_job_run_id": "string", + "context_sources_task_run_id": "string", + "context_sources_version": "string", + "context_user_agent": "string", + "currency": "string", + "event": "string", + "id": "string", + "p_map_nested_map": "json", + "received_at": "datetime", + "revenue": "int", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + bq:[ + { + "data": { + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_sources_job_id": "djfhksdjhfkjdhfkjahkf", + "context_sources_job_run_id": "job_run_id", + "context_sources_task_run_id": "task_run_id", + "context_sources_version": "1169/merge", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "event": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50 + }, + "metadata": { + "columns": { + "context_c_map_nested_map": "string", + "context_sources_job_id": "string", + "context_sources_job_run_id": "string", + "context_sources_task_run_id": "string", + "context_sources_version": "string", + "context_user_agent": "string", + "currency": "string", + "event": "string", + "id": "string", + "loaded_at": "datetime", + "p_map_nested_map": "string", + "received_at": "datetime", + "revenue": "int", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + postgres: [ + { + "data": { + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_sources_job_id": "djfhksdjhfkjdhfkjahkf", + "context_sources_job_run_id": "job_run_id", + "context_sources_task_run_id": "task_run_id", + "context_sources_version": "1169/merge", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "event": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50 + }, + "metadata": { + "columns": { + "context_c_map_nested_map": "json", + "context_sources_job_id": "string", + "context_sources_job_run_id": "string", + "context_sources_task_run_id": "string", + "context_sources_version": "string", + "context_user_agent": "string", + "currency": "string", + "event": "string", + "id": "string", + "p_map_nested_map": "json", + "received_at": "datetime", + "revenue": "int", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + snowflake: [ + { + "data": { + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_SOURCES_JOB_ID": "djfhksdjhfkjdhfkjahkf", + "CONTEXT_SOURCES_JOB_RUN_ID": "job_run_id", + "CONTEXT_SOURCES_TASK_RUN_ID": "task_run_id", + "CONTEXT_SOURCES_VERSION": "1169/merge", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EVENT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50 + }, + "metadata": { + "columns": { + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_SOURCES_JOB_ID": "string", + "CONTEXT_SOURCES_JOB_RUN_ID": "string", + "CONTEXT_SOURCES_TASK_RUN_ID": "string", + "CONTEXT_SOURCES_VERSION": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EVENT": "string", + "ID": "string", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } + ] + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js new file mode 100644 index 0000000000..f84f9d33ed --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/groups.js @@ -0,0 +1,420 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey", + testMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + ctestMap: { + cnestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-GB", + screen: { + density: 2 + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true, + RS: { + options: { + jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] + } + }, + BQ: { + options: { + jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] + } + }, + POSTGRES: { + options: { + jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: ["group.traits.testMap.nestedMap", "group.context.ctestMap.cnestedMap"] + } + }, + GCS_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "group", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "groups" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map": "json", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "groups" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "_groups" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "city": "Disney", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "country": "USA", + "email": "mickey@disney.com", + "firstname": "Mickey", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "city": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "country": "string", + "email": "string", + "firstname": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "test_map_nested_map": "json", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "groups" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CITY": "Disney", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "COUNTRY": "USA", + "EMAIL": "mickey@disney.com", + "FIRSTNAME": "Mickey", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CITY": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "COUNTRY": "string", + "EMAIL": "string", + "FIRSTNAME": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "GROUPS" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js new file mode 100644 index 0000000000..3d6164b430 --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/identifies.js @@ -0,0 +1,850 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + type: "identify", + sentAt: "2021-01-03T17:02:53.195Z", + userId: "user123", + channel: "web", + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "identify.userProperties.UPMap.nestedMap", + "identify.context.traits.CTMap.nestedMap", + "identify.traits.TMap.nestedMap", + "identify.context.CMap.nestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "identify.userProperties.UPMap.nestedMap", + "identify.context.traits.CTMap.nestedMap", + "identify.traits.TMap.nestedMap", + "identify.context.CMap.nestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "identify.userProperties.UPMap.nestedMap", + "identify.context.traits.CTMap.nestedMap", + "identify.traits.TMap.nestedMap", + "identify.context.CMap.nestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "identify.userProperties.UPMap.nestedMap", + "identify.context.traits.CTMap.nestedMap", + "identify.traits.TMap.nestedMap", + "identify.context.CMap.nestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + userProperties: { + email: "test@gmail.com", + UPMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + traits: { + TMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + context: { + os: { + "name": "android", + "version": "1.12.3" + }, + app: { + name: "RudderLabs JavaScript SDK", + build: "1.0.0", + version: "1.1.11", + namespace: "com.rudderlabs.javascript" + }, + traits: { + email: "user123@email.com", + phone: "+917836362334", + userId: "user123", + CTMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + CMap: { + nestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-US", + device: { + token: "token", + id: "id", + type: "ios" + }, + library: { + name: "RudderLabs JavaScript SDK", + version: "1.1.11" + }, + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0" + }, + rudderId: "8f8fa6b5-8e24-489c-8e22-61f23f2e364f", + messageId: "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + anonymousId: "97c46c81-3140-456d-b2a9-690d70aaca35", + originalTimestamp: "2020-01-24T06:29:02.364Z", + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + timestamp: "2020-01-24T11:59:02.403+05:30" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map_n_1": "string", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map_n_1": "nested prop 1", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map_n_1": "nested prop 1", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map_n_1": "nested prop 1", + "up_map_nested_map_n_1": "nested prop 1" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map_n_1": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map_n_1": "string", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map_n_1": "string", + "up_map_nested_map_n_1": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "json", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map": "json", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map": "json", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map": "json", + "timestamp": "datetime", + "up_map_nested_map": "json", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "json", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map": "json", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map": "json", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map": "json", + "up_map_nested_map": "json", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map": "string", + "email": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map": "string", + "timestamp": "datetime", + "up_map_nested_map": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "string", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map": "string", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map": "string", + "email": "string", + "id": "string", + "loaded_at": "datetime", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map": "string", + "up_map_nested_map": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "97c46c81-3140-456d-b2a9-690d70aaca35", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "email": "user123@email.com", + "id": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2021-01-03T17:02:53.195Z", + "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "user123" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "json", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map": "json", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map": "json", + "email": "string", + "id": "string", + "original_timestamp": "datetime", + "phone": "string", + "received_at": "datetime", + "sent_at": "datetime", + "t_map_nested_map": "json", + "timestamp": "datetime", + "up_map_nested_map": "json", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "identifies" + } + }, + { + "data": { + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.1.11", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_device_id": "id", + "context_device_token": "token", + "context_device_type": "ios", + "context_ip": "[::1]:53708", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.1.11", + "context_locale": "en-US", + "context_os_name": "android", + "context_os_version": "1.12.3", + "context_request_ip": "[::1]:53708", + "context_traits_ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "context_traits_email": "user123@email.com", + "context_traits_phone": "+917836362334", + "context_traits_user_id": "user123", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "ct_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "email": "user123@email.com", + "id": "user123", + "phone": "+917836362334", + "received_at": "2020-01-24T06:29:02.403Z", + "t_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}" + }, + "metadata": { + "columns": { + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "json", + "context_device_id": "string", + "context_device_token": "string", + "context_device_type": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_os_name": "string", + "context_os_version": "string", + "context_request_ip": "string", + "context_traits_ct_map_nested_map": "json", + "context_traits_email": "string", + "context_traits_phone": "string", + "context_traits_user_id": "string", + "context_user_agent": "string", + "ct_map_nested_map": "json", + "email": "string", + "id": "string", + "phone": "string", + "received_at": "datetime", + "t_map_nested_map": "json", + "up_map_nested_map": "json", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "users" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "97c46c81-3140-456d-b2a9-690d70aaca35", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "EMAIL": "user123@email.com", + "ID": "2116ef8c-efc3-4ca4-851b-02ee60dad6ff", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2021-01-03T17:02:53.195Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "user123" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "json", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP": "json", + "EMAIL": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "T_MAP_NESTED_MAP": "json", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "IDENTIFIES" + } + }, + { + "data": { + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.1.11", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_DEVICE_ID": "id", + "CONTEXT_DEVICE_TOKEN": "token", + "CONTEXT_DEVICE_TYPE": "ios", + "CONTEXT_IP": "[::1]:53708", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.1.11", + "CONTEXT_LOCALE": "en-US", + "CONTEXT_OS_NAME": "android", + "CONTEXT_OS_VERSION": "1.12.3", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "CONTEXT_TRAITS_EMAIL": "user123@email.com", + "CONTEXT_TRAITS_PHONE": "+917836362334", + "CONTEXT_TRAITS_USER_ID": "user123", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0", + "CT_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "EMAIL": "user123@email.com", + "ID": "user123", + "PHONE": "+917836362334", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "T_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}" + }, + "metadata": { + "columns": { + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_DEVICE_ID": "string", + "CONTEXT_DEVICE_TOKEN": "string", + "CONTEXT_DEVICE_TYPE": "string", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_OS_NAME": "string", + "CONTEXT_OS_VERSION": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_TRAITS_CT_MAP_NESTED_MAP": "json", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_PHONE": "string", + "CONTEXT_TRAITS_USER_ID": "string", + "CONTEXT_USER_AGENT": "string", + "CT_MAP_NESTED_MAP": "json", + "EMAIL": "string", + "ID": "string", + "PHONE": "string", + "RECEIVED_AT": "datetime", + "T_MAP_NESTED_MAP": "json", + "UP_MAP_NESTED_MAP": "json", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "USERS" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js new file mode 100644 index 0000000000..136f355b21 --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/pages.js @@ -0,0 +1,393 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + ctestMap: { + cnestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-GB", + screen: { + density: 2 + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true, + RS: { + options: { + jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] + } + }, + BQ: { + options: { + jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] + } + }, + POSTGRES: { + options: { + jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: ["page.properties.testMap.nestedMap", "page.context.ctestMap.cnestedMap"] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { + currency: "USD", + revenue: 50, + testMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "page", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "pages" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map": "json", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "pages" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "pages" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map": "json", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "pages" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PAGES" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js new file mode 100644 index 0000000000..b11b311ebb --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/screens.js @@ -0,0 +1,393 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + ctestMap: { + cnestedMap: { + n1: "context nested prop 1" + } + }, + locale: "en-GB", + screen: { + density: 2 + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + integrations: { + All: true, + RS: { + options: { + jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] + } + }, + BQ: { + options: { + jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] + } + }, + POSTGRES: { + options: { + jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: ["screen.properties.testMap.nestedMap", "screen.context.ctestMap.cnestedMap"] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + properties: { + currency: "USD", + revenue: 50, + testMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "screen", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map_n_1": "nested prop 1", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map_n_1": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "screens" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map": "json", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "screens" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map": "string", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "screens" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_ctest_map_cnested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "test_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_ctest_map_cnested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_user_agent": "string", + "currency": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "test_map_nested_map": "json", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "screens" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TEST_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_CTEST_MAP_CNESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TEST_MAP_NESTED_MAP": "json", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "SCREENS" + } + } + ], + } +} diff --git a/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js new file mode 100644 index 0000000000..7ed95685ff --- /dev/null +++ b/test/__tests__/data/warehouse/integrations/jsonpaths/new/tracks.js @@ -0,0 +1,830 @@ +module.exports = { + input: { + destination: { + Config: {} + }, + message: { + anonymousId: "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + channel: "web", + context: { + app: { + build: "1.0.0", + name: "RudderLabs JavaScript SDK", + namespace: "com.rudderlabs.javascript", + version: "1.0.5" + }, + ip: "0.0.0.0", + library: { + name: "RudderLabs JavaScript SDK", + version: "1.0.5" + }, + locale: "en-GB", + screen: { + density: 2 + }, + traits: { + city: "Disney", + country: "USA", + email: "mickey@disney.com", + firstname: "Mickey" + }, + CMap: { + nestedMap: { + n1: "context nested prop 1" + } + }, + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36" + }, + event: "Product Added", + integrations: { + All: true, + RS: { + options: { + jsonPaths: [ + "track.userProperties.UPMap.nestedMap", + "track.properties.PMap.nestedMap", + "track.context.CMap.nestedMap", + ] + } + }, + BQ: { + options: { + jsonPaths: [ + "track.userProperties.UPMap.nestedMap", + "track.properties.PMap.nestedMap", + "track.context.CMap.nestedMap", + ] + } + }, + POSTGRES: { + options: { + jsonPaths: [ + "track.userProperties.UPMap.nestedMap", + "track.properties.PMap.nestedMap", + "track.context.CMap.nestedMap", + ] + } + }, + SNOWFLAKE: { + options: { + jsonPaths: [ + "track.userProperties.UPMap.nestedMap", + "track.properties.PMap.nestedMap", + "track.context.CMap.nestedMap", + ] + } + }, + S3_DATALAKE: { + options: { + skipReservedKeywordsEscaping: true + } + }, + }, + messageId: "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + originalTimestamp: "2020-01-24T06:29:02.364Z", + userProperties: { + email: "test@gmail.com", + UPMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + properties: { + currency: "USD", + revenue: 50, + PMap: { + nestedMap: { + n1: "nested prop 1" + } + }, + }, + receivedAt: "2020-01-24T11:59:02.403+05:30", + request_ip: "[::1]:53708", + sentAt: "2020-01-24T06:29:02.364Z", + timestamp: "2020-01-24T11:59:02.403+05:30", + type: "track", + userId: "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + request: { + query: { + whSchemaVersion: "v1" + } + } + }, + output: { + default: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "tracks" + } + }, + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map_n_1": "context nested prop 1", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "email": "test@gmail.com", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "p_map_nested_map_n_1": "nested prop 1", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map_n_1": "nested prop 1", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map_n_1": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "currency": "string", + "email": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "p_map_nested_map_n_1": "string", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "timestamp": "datetime", + "up_map_nested_map_n_1": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + rs: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "tracks" + } + }, + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "email": "test@gmail.com", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "currency": "string", + "email": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "p_map_nested_map": "json", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "timestamp": "datetime", + "up_map_nested_map": "json", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + bq: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "event": "string", + "event_text": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "tracks" + } + }, + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "email": "test@gmail.com", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "string", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "currency": "string", + "email": "string", + "event": "string", + "event_text": "string", + "id": "string", + "loaded_at": "datetime", + "original_timestamp": "datetime", + "p_map_nested_map": "string", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "timestamp": "datetime", + "up_map_nested_map": "string", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + postgres: [ + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "received_at": "2020-01-24T06:29:02.403Z", + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "received_at": "datetime", + "sent_at": "datetime", + "timestamp": "datetime", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "tracks" + } + }, + { + "data": { + "anonymous_id": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "channel": "web", + "context_app_build": "1.0.0", + "context_app_name": "RudderLabs JavaScript SDK", + "context_app_namespace": "com.rudderlabs.javascript", + "context_app_version": "1.0.5", + "context_c_map_nested_map": "{\"n1\":\"context nested prop 1\"}", + "context_ip": "0.0.0.0", + "context_library_name": "RudderLabs JavaScript SDK", + "context_library_version": "1.0.5", + "context_locale": "en-GB", + "context_passed_ip": "0.0.0.0", + "context_request_ip": "[::1]:53708", + "context_screen_density": 2, + "context_traits_city": "Disney", + "context_traits_country": "USA", + "context_traits_email": "mickey@disney.com", + "context_traits_firstname": "Mickey", + "context_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "currency": "USD", + "email": "test@gmail.com", + "event": "product_added", + "event_text": "Product Added", + "id": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "original_timestamp": "2020-01-24T06:29:02.364Z", + "p_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "received_at": "2020-01-24T06:29:02.403Z", + "revenue": 50, + "sent_at": "2020-01-24T06:29:02.364Z", + "timestamp": "2020-01-24T06:29:02.403Z", + "up_map_nested_map": "{\"n1\":\"nested prop 1\"}", + "user_id": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "anonymous_id": "string", + "channel": "string", + "context_app_build": "string", + "context_app_name": "string", + "context_app_namespace": "string", + "context_app_version": "string", + "context_c_map_nested_map": "json", + "context_ip": "string", + "context_library_name": "string", + "context_library_version": "string", + "context_locale": "string", + "context_passed_ip": "string", + "context_request_ip": "string", + "context_screen_density": "int", + "context_traits_city": "string", + "context_traits_country": "string", + "context_traits_email": "string", + "context_traits_firstname": "string", + "context_user_agent": "string", + "currency": "string", + "email": "string", + "event": "string", + "event_text": "string", + "id": "string", + "original_timestamp": "datetime", + "p_map_nested_map": "json", + "received_at": "datetime", + "revenue": "int", + "sent_at": "datetime", + "timestamp": "datetime", + "up_map_nested_map": "json", + "user_id": "string", + "uuid_ts": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "product_added" + } + } + ], + snowflake: [ + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "RECEIVED_AT": "datetime", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "TRACKS" + } + }, + { + "data": { + "ANONYMOUS_ID": "e6ab2c5e-2cda-44a9-a962-e2f67df78bca", + "CHANNEL": "web", + "CONTEXT_APP_BUILD": "1.0.0", + "CONTEXT_APP_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_APP_NAMESPACE": "com.rudderlabs.javascript", + "CONTEXT_APP_VERSION": "1.0.5", + "CONTEXT_C_MAP_NESTED_MAP": "{\"n1\":\"context nested prop 1\"}", + "CONTEXT_IP": "0.0.0.0", + "CONTEXT_LIBRARY_NAME": "RudderLabs JavaScript SDK", + "CONTEXT_LIBRARY_VERSION": "1.0.5", + "CONTEXT_LOCALE": "en-GB", + "CONTEXT_PASSED_IP": "0.0.0.0", + "CONTEXT_REQUEST_IP": "[::1]:53708", + "CONTEXT_SCREEN_DENSITY": 2, + "CONTEXT_TRAITS_CITY": "Disney", + "CONTEXT_TRAITS_COUNTRY": "USA", + "CONTEXT_TRAITS_EMAIL": "mickey@disney.com", + "CONTEXT_TRAITS_FIRSTNAME": "Mickey", + "CONTEXT_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36", + "CURRENCY": "USD", + "EMAIL": "test@gmail.com", + "EVENT": "product_added", + "EVENT_TEXT": "Product Added", + "ID": "a6a0ad5a-bd26-4f19-8f75-38484e580fc7", + "ORIGINAL_TIMESTAMP": "2020-01-24T06:29:02.364Z", + "P_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "RECEIVED_AT": "2020-01-24T06:29:02.403Z", + "REVENUE": 50, + "SENT_AT": "2020-01-24T06:29:02.364Z", + "TIMESTAMP": "2020-01-24T06:29:02.403Z", + "UP_MAP_NESTED_MAP": "{\"n1\":\"nested prop 1\"}", + "USER_ID": "9bb5d4c2-a7aa-4a36-9efb-dd2b1aec5d33" + }, + "metadata": { + "columns": { + "ANONYMOUS_ID": "string", + "CHANNEL": "string", + "CONTEXT_APP_BUILD": "string", + "CONTEXT_APP_NAME": "string", + "CONTEXT_APP_NAMESPACE": "string", + "CONTEXT_APP_VERSION": "string", + "CONTEXT_C_MAP_NESTED_MAP": "json", + "CONTEXT_IP": "string", + "CONTEXT_LIBRARY_NAME": "string", + "CONTEXT_LIBRARY_VERSION": "string", + "CONTEXT_LOCALE": "string", + "CONTEXT_PASSED_IP": "string", + "CONTEXT_REQUEST_IP": "string", + "CONTEXT_SCREEN_DENSITY": "int", + "CONTEXT_TRAITS_CITY": "string", + "CONTEXT_TRAITS_COUNTRY": "string", + "CONTEXT_TRAITS_EMAIL": "string", + "CONTEXT_TRAITS_FIRSTNAME": "string", + "CONTEXT_USER_AGENT": "string", + "CURRENCY": "string", + "EMAIL": "string", + "EVENT": "string", + "EVENT_TEXT": "string", + "ID": "string", + "ORIGINAL_TIMESTAMP": "datetime", + "P_MAP_NESTED_MAP": "json", + "RECEIVED_AT": "datetime", + "REVENUE": "int", + "SENT_AT": "datetime", + "TIMESTAMP": "datetime", + "UP_MAP_NESTED_MAP": "json", + "USER_ID": "string", + "UUID_TS": "datetime" + }, + "receivedAt": "2020-01-24T11:59:02.403+05:30", + "table": "PRODUCT_ADDED" + } + } + ] + } +} diff --git a/test/__tests__/flatten_events.test.js b/test/__tests__/flatten_events.test.js index 2e3f142aca..502140bccd 100644 --- a/test/__tests__/flatten_events.test.js +++ b/test/__tests__/flatten_events.test.js @@ -9,7 +9,7 @@ Key aspects covered: 2. Coud source event - same as above for nested property under level 3 - Above level 3 are stringified and considered as string type - 3. Introducing jsonKeys for BQ, PG, RS, SF + 3. Introducing jsonLegacyPathKeys for BQ, PG, RS, SF - data types: int, float, string, json - number, string, array, objects fit under json data type, value is json stringified - in BQ, json type is just a string data type @@ -83,8 +83,8 @@ describe("Flatten event properties", () => { expect(columnTypes).toEqual(expected.columnTypes); }); - it("Should stringify properties of sample event if jsonKeys are present", () => { - options.jsonKeys = { + it("Should stringify properties of sample event if jsonLegacyPathKeys are present", () => { + options.jsonLegacyPathKeys = { arrayProp: 0, objectProp_firstLevelMap_secondLevelMap: 2 }; @@ -98,14 +98,14 @@ describe("Flatten event properties", () => { columnTypes, options ); - delete options.jsonKeys; + delete options.jsonLegacyPathKeys; const expected = fOutput("json_key_event", integrations[index]); expect(output).toEqual(expected.output); expect(columnTypes).toEqual(expected.columnTypes); }); - it("Should stringify properties of sample event and jsonKeys has more priority than cloud source at level 3", () => { - options.jsonKeys = { + it("Should stringify properties of sample event and jsonLegacyPathKeys has more priority than cloud source at level 3", () => { + options.jsonLegacyPathKeys = { objectProp_firstLevelMap_secondLevelMap_thirdLevelMap: 3 }; options.sourceCategory = "cloud"; @@ -119,15 +119,15 @@ describe("Flatten event properties", () => { columnTypes, options ); - delete options.jsonKeys; + delete options.jsonLegacyPathKeys; delete options.sourceCategory; const expected = fOutput("json_key_cloud_event", integrations[index]); expect(output).toEqual(expected.output); expect(columnTypes).toEqual(expected.columnTypes); }); - it("Should stringify properties of nested event and jsonKeys is not applied for cloud source at level > 3", () => { - options.jsonKeys = { + it("Should stringify properties of nested event and jsonLegacyPathKeys is not applied for cloud source at level > 3", () => { + options.jsonLegacyPathKeys = { objectProp_firstLevelMap_secondLevelMap_thirdLevelMap_fourthLevelMap: 4, arrayProp: 0 }; @@ -142,15 +142,15 @@ describe("Flatten event properties", () => { columnTypes, options ); - delete options.jsonKeys; + delete options.jsonLegacyPathKeys; delete options.sourceCategory; const expected = fOutput("cloud_json_key_event", integrations[index]); expect(output).toEqual(expected.output); expect(columnTypes).toEqual(expected.columnTypes); }); - it("Should flatten all properties of sample event and jsonkeys is not applicable other than track events", () => { - options.jsonKeys = { arrayProp: 0 }; + it("Should flatten all properties of sample event and jsonLegacyPathKeys is not applicable other than track events", () => { + options.jsonLegacyPathKeys = { arrayProp: 0 }; const output = {}; const columnTypes = {}; setDataFromInputAndComputeColumnTypes( @@ -161,19 +161,19 @@ describe("Flatten event properties", () => { columnTypes, options ); - delete options.jsonKeys; + delete options.jsonLegacyPathKeys; // Will be same as flattening all properties const expected = fOutput("sample_event", integrations[index]); expect(output).toEqual(expected.output); expect(columnTypes).toEqual(expected.columnTypes); }); - it("Should flatten all properties of sample event and declared jsonKeys get stringified primitive values", () => { + it("Should flatten all properties of sample event and declared jsonLegacyPathKeys get stringified primitive values", () => { i.dateProp = "2022-01-01T00:00:00.000Z"; i.objectProp.firstLevelDateProp = "2022-01-01T01:01:01.111Z"; i.objectProp.firstLevelMap.secondLevelDateProp = "2022-01-01T02:02:02.222Z"; - options.jsonKeys = { + options.jsonLegacyPathKeys = { nullProp: 0, blankProp: 0, floatProp: 0, @@ -195,7 +195,7 @@ describe("Flatten event properties", () => { columnTypes, options ); - delete options.jsonKeys; + delete options.jsonLegacyPathKeys; delete i.dateProp; delete i.objectProp.firstLevelDateProp; delete i.objectProp.firstLevelMap.secondLevelDateProp; @@ -204,12 +204,12 @@ describe("Flatten event properties", () => { expect(columnTypes).toEqual(expected.columnTypes); }); - it("Should ignore rudder reserved columns even though they are set as jsonKeys", () => { + it("Should ignore rudder reserved columns even though they are set as jsonLegacyPathKeys", () => { // setting two reserved columns as map i.anonymous_id = "ignored column"; i.timestamp = "ignored column"; // setting reserved colum paths as json keys - options.jsonKeys = { anonymous_id: 0, timestamp: 0 }; + options.jsonLegacyPathKeys = { anonymous_id: 0, timestamp: 0 }; const output = {}; const columnTypes = {}; setDataFromInputAndComputeColumnTypes( @@ -220,19 +220,19 @@ describe("Flatten event properties", () => { columnTypes, options ); - delete options.jsonKeys; + delete options.jsonLegacyPathKeys; // Will be same as flattening all properties with reserved columns being ignored const expected = fOutput("sample_event", integrations[index]); expect(output).toEqual(expected.output); expect(columnTypes).toEqual(expected.columnTypes); }); - it("Should apply escaping on WH reserved columns even though they are set as jsonKeys", () => { + it("Should apply escaping on WH reserved columns even though they are set as jsonLegacyPathKeys", () => { // setting two reserved columns as map i.between = "escaped column"; i.as = "escaped column"; // setting reserved colum paths as json keys - options.jsonKeys = { between: 0, as: 0 }; + options.jsonLegacyPathKeys = { between: 0, as: 0 }; const output = {}; const columnTypes = {}; setDataFromInputAndComputeColumnTypes( @@ -243,7 +243,7 @@ describe("Flatten event properties", () => { columnTypes, options ); - delete options.jsonKeys; + delete options.jsonLegacyPathKeys; // Escaping is applied as usual for json keys too const expected = fOutput("escape_event", integrations[index]); expect(output).toEqual(expected.output); diff --git a/test/__tests__/warehouse.test.js b/test/__tests__/warehouse.test.js index 6d8b490208..045bab35a6 100644 --- a/test/__tests__/warehouse.test.js +++ b/test/__tests__/warehouse.test.js @@ -30,7 +30,10 @@ const integrations = [ "snowflake", "mssql", "azure_synapse", - "s3_datalake" + "deltalake", + "azure_datalake", + "s3_datalake", + "gcs_datalake", ]; const transformers = integrations.map(integration => require(`../../src/${version}/destinations/${integration}/transform`) @@ -229,7 +232,7 @@ describe("column & table names", () => { ); return; } - if (integrations[index] === "s3_datalake") { + if (integrations[index] === "s3_datalake" || integrations[index] === "gcs_datalake" || integrations[index] === "azure_datalake") { expect(received[1].metadata).toHaveProperty( "table", "a_1_a_2_a_3_a_4_a_5_b_1_b_2_b_3_b_4_b_5_c_1_c_2_c_3_c_4_c_5_d_1_d_2_d_3_d_4_d_5_e_1_e_2_e_3_e_4_e_5_f_1_f_2_f_3_f_4_f_5_g_1_g_2_g_3_g_4_g_5" @@ -943,7 +946,7 @@ describe("Handle no of columns in an event", () => { it("should throw an error if no of columns are more than 200", () => { const i = input("track"); transformers - .filter((transformer, index) => integrations[index] !== "s3_datalake") + .filter((transformer, index) => integrations[index] !== "s3_datalake" && integrations[index] !== "gcs_datalake" && integrations[index] !== "azure_datalake") .forEach((transformer, index) => { i.message.properties = largeNoOfColumnsevent; expect(() => transformer.process(i)).toThrow( @@ -1009,7 +1012,7 @@ describe("Integration options", () => { it("should generate two events for every track call", () => { const i = opInput("track"); transformers.forEach((transformer, index) => { - const { jsonPaths } = i.destination.Config; + const {jsonPaths} = i.destination.Config; if (integrations[index] === "postgres") { delete i.destination.Config.jsonPaths; } @@ -1019,6 +1022,7 @@ describe("Integration options", () => { }); }); }); + describe("users", () => { it("should skip users when skipUsersTable is set", () => { const i = opInput("users"); @@ -1028,6 +1032,65 @@ describe("Integration options", () => { }); }); }); + + describe("json paths", () => { + const output = (config, provider) => { + switch (provider) { + case "rs": + return _.cloneDeep(config.output.rs); + case "bq": + return _.cloneDeep(config.output.bq); + case "postgres": + return _.cloneDeep(config.output.postgres); + case "snowflake": + return _.cloneDeep(config.output.snowflake); + default: + return _.cloneDeep(config.output.default); + } + } + + const testCases = [ + { + eventType: "aliases", + }, + { + eventType: "groups", + }, + { + eventType: "identifies", + }, + { + eventType: "pages", + }, + { + eventType: "screens", + }, + { + eventType: "tracks", + }, + { + eventType: "extract", + }, + ]; + + for (const testCase of testCases) { + transformers.forEach((transformer, index) => { + it(`new ${testCase.eventType} for ${integrations[index]}`, () => { + const config = require("./data/warehouse/integrations/jsonpaths/new/" + testCase.eventType); + const input = _.cloneDeep(config.input); + const received = transformer.process(input); + expect(received).toEqual(output(config, integrations[index])); + }) + + it(`legacy ${testCase.eventType} for ${integrations[index]}`, () => { + const config = require("./data/warehouse/integrations/jsonpaths/legacy/" + testCase.eventType); + const input = _.cloneDeep(config.input); + const received = transformer.process(input); + expect(received).toEqual(output(config, integrations[index])); + }) + }); + } + }); }); describe("validTimestamp", () => { From 266514bd56c150d6c88c1db0733a1da0a4367c02 Mon Sep 17 00:00:00 2001 From: mihir-4116 Date: Mon, 11 Sep 2023 10:52:16 +0530 Subject: [PATCH 3/5] fix(bugsnag): alerts --- src/v0/destinations/adobe_analytics/transform.js | 6 +++--- src/v0/destinations/branch/transform.js | 6 +++++- src/v0/destinations/fb/transform.js | 12 ++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/v0/destinations/adobe_analytics/transform.js b/src/v0/destinations/adobe_analytics/transform.js index 5daf11e62e..c477b4c93c 100644 --- a/src/v0/destinations/adobe_analytics/transform.js +++ b/src/v0/destinations/adobe_analytics/transform.js @@ -341,7 +341,7 @@ const handleTrack = (message, destinationConfig) => { let payload = null; // handle ecommerce events separately // generic events should go to the default - const event = rawEvent?.toLowerCase(); + const event = typeof rawEvent === 'string' ? rawEvent.toLowerCase() : rawEvent; switch (event) { case 'product viewed': case 'product list viewed': @@ -372,10 +372,10 @@ const handleTrack = (message, destinationConfig) => { payload = processTrackEvent(message, 'scOpen', destinationConfig); break; default: - if (destinationConfig.rudderEventsToAdobeEvents[event.toLowerCase()]) { + if (destinationConfig.rudderEventsToAdobeEvents[event]) { payload = processTrackEvent( message, - destinationConfig.rudderEventsToAdobeEvents[event.toLowerCase()].trim(), + destinationConfig.rudderEventsToAdobeEvents[event]?.trim(), destinationConfig, ); } else if (message?.properties?.overrideEventName) { diff --git a/src/v0/destinations/branch/transform.js b/src/v0/destinations/branch/transform.js index ae214819b2..362f6cc840 100644 --- a/src/v0/destinations/branch/transform.js +++ b/src/v0/destinations/branch/transform.js @@ -46,7 +46,11 @@ function getCategoryAndName(rudderEventName) { let requiredCategory = null; // eslint-disable-next-line array-callback-return Object.keys(category.name).find((branchKey) => { - if (branchKey.toLowerCase() === rudderEventName.toLowerCase()) { + if ( + typeof branchKey === 'string' && + typeof rudderEventName === 'string' && + branchKey.toLowerCase() === rudderEventName.toLowerCase() + ) { requiredName = category.name[branchKey]; requiredCategory = category; } diff --git a/src/v0/destinations/fb/transform.js b/src/v0/destinations/fb/transform.js index a73bbdca8b..a6a8b71e8d 100644 --- a/src/v0/destinations/fb/transform.js +++ b/src/v0/destinations/fb/transform.js @@ -83,7 +83,11 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { case 'ud[ln]': case 'ud[st]': case 'ud[cn]': - if (clonedUpdatedEvent[prop] && clonedUpdatedEvent[prop] !== '') { + if ( + clonedUpdatedEvent[prop] && + typeof clonedUpdatedEvent[prop] === 'string' && + clonedUpdatedEvent[prop] !== '' + ) { isUDSet = true; clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].toLowerCase()); } @@ -105,8 +109,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { if (clonedUpdatedEvent[prop] && clonedUpdatedEvent[prop] !== '') { if (typeof clonedUpdatedEvent[prop] !== 'string') { delete clonedUpdatedEvent[prop]; - } - else { + } else { isUDSet = true; clonedUpdatedEvent[prop] = sha256( clonedUpdatedEvent[prop].toLowerCase() === 'female' ? 'f' : 'm', @@ -167,7 +170,8 @@ function getCorrectedTypedValue(pathToKey, value, originalPath) { } throw new InstrumentationError( - `${typeof originalPath === 'object' ? JSON.stringify(originalPath) : originalPath + `${ + typeof originalPath === 'object' ? JSON.stringify(originalPath) : originalPath } is not of valid type`, ); } From f0f20d654ec5ee8eb078ce7f5610a4666d73fd8c Mon Sep 17 00:00:00 2001 From: Ujjwal Abhishek <63387036+ujjwal-ab@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:59:16 +0530 Subject: [PATCH 4/5] feat: add support for updating profile properties through track call (#2581) add all traits to profile properties in track call --- src/v0/destinations/klaviyo/transform.js | 31 ++++++------ src/v0/destinations/klaviyo/util.js | 2 +- test/__tests__/data/klaviyo.json | 62 +++++++++++++----------- 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/src/v0/destinations/klaviyo/transform.js b/src/v0/destinations/klaviyo/transform.js index d7170ce789..bb19c5f8fd 100644 --- a/src/v0/destinations/klaviyo/transform.js +++ b/src/v0/destinations/klaviyo/transform.js @@ -42,9 +42,10 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); /** * Main Identify request handler func * The function is used to create/update new users and also for adding/subscribing - * members to the list depending on conditons.If listId is there member is added to that list & - * if subscribe is true member is subscribed to that list else not. - * DOCS: https://www.klaviyo.com/docs/http-api + * users to the list. + * DOCS: 1. https://developers.klaviyo.com/en/v2023-02-22/reference/create_profile + * 2. https://developers.klaviyo.com/en/v2023-02-22/reference/update_profile + * 3. https://developers.klaviyo.com/en/v2023-02-22/reference/subscribe_profiles * @param {*} message * @param {*} category * @param {*} destination @@ -123,7 +124,7 @@ const identifyRequestHandler = async (message, category, destination) => { // ---------------------- // Main handler func for track request/screen request // User info needs to be mapped to a track event (mandatory) -// DOCS: https://www.klaviyo.com/docs/http-api +// DOCS: https://developers.klaviyo.com/en/v2023-02-22/reference/create_event // ---------------------- const trackRequestHandler = (message, category, destination) => { @@ -138,10 +139,6 @@ const trackRequestHandler = (message, category, destination) => { attributes.metric = { name: eventName }; const categ = CONFIG_CATEGORIES[eventMap]; attributes.properties = constructPayload(message.properties, MAPPING_CONFIG[categ.name]); - attributes.properties = { - ...attributes.properties, - ...populateCustomFieldsFromTraits(message), - }; // products mapping using Items.json // mapping properties.items to payload.properties.items and using properties.products as a fallback to properties.items @@ -191,17 +188,21 @@ const trackRequestHandler = (message, category, destination) => { if (value) { attributes.value = value; } - attributes.properties = { - ...attributes.properties, - ...populateCustomFieldsFromTraits(message), - }; } // if flattenProperties is enabled from UI, flatten the event properties attributes.properties = flattenProperties ? flattenJson(attributes.properties, '.', 'normal', false) : attributes.properties; // Map user properties to profile object - attributes.profile = createCustomerProperties(message, destination.Config); + attributes.profile = { + ...createCustomerProperties(message, destination.Config), + ...populateCustomFieldsFromTraits(message), + }; + + attributes.profile = flattenProperties + ? flattenJson(attributes.profile, '.', 'normal', false) + : attributes.profile; + if (message.timestamp) { attributes.time = message.timestamp; } @@ -222,9 +223,9 @@ const trackRequestHandler = (message, category, destination) => { // ---------------------- // Main handlerfunc for group request -// we will map user to list (subscribe and/or member) +// we will add/subscribe users to the list // based on property sent -// DOCS: https://www.klaviyo.com/docs/api/v2/lists +// DOCS: https://developers.klaviyo.com/en/v2023-02-22/reference/subscribe_profiles // ---------------------- const groupRequestHandler = (message, category, destination) => { if (!message.groupId) { diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 8fb9f0e875..3c4882b834 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -96,7 +96,7 @@ const profileUpdateResponseBuilder = (payload, profileId, category, privateApiKe /** * This function is used for creating response for subscribing users to a particular list. - * DOCS: https://www.klaviyo.com/docs/api/v2/lists + * DOCS: https://developers.klaviyo.com/en/v2023-02-22/reference/subscribe_profiles */ const subscribeUserToList = (message, traitsInfo, destination) => { // listId from message properties are preferred over Config listId diff --git a/test/__tests__/data/klaviyo.json b/test/__tests__/data/klaviyo.json index e3ce30c792..1fc46a6e4c 100644 --- a/test/__tests__/data/klaviyo.json +++ b/test/__tests__/data/klaviyo.json @@ -705,7 +705,7 @@ } }, { - "description": "Track event call", + "description": "Screen event call", "input": { "destination": { "Config": { @@ -800,13 +800,13 @@ "properties": { "PreviouslyVicePresident": true, "YearElected": 1801, - "VicePresidents": ["Aaron Burr", "George Clinton"], - "age": "22" + "VicePresidents": ["Aaron Burr", "George Clinton"] }, "profile": { "$email": "test@rudderstack.com", "$phone_number": "9112340375", - "$id": "sajal12" + "$id": "sajal12", + "age": "22" } } } @@ -863,7 +863,11 @@ "age": "22", "email": "test@rudderstack.com", "phone": "9112340375", - "anonymousId": "9c6bd77ea9da3e68" + "anonymousId": "9c6bd77ea9da3e68", + "plan_details": { + "plan_type": "gold", + "duration": "3 months" + } }, "library": { "name": "com.rudderstack.android.sdk.core", @@ -917,13 +921,15 @@ "properties": { "vicePresdentInfo.PreviouslVicePresident": true, "vicePresdentInfo.VicePresidents": ["AaronBurr", "GeorgeClinton"], - "vicePresdentInfo.YearElected": 1801, - "age": "22" + "vicePresdentInfo.YearElected": 1801 }, "profile": { "$email": "test@rudderstack.com", "$phone_number": "9112340375", - "$id": "sajal12" + "$id": "sajal12", + "age": "22", + "plan_details.plan_type": "gold", + "plan_details.duration": "3 months" } } } @@ -1034,15 +1040,15 @@ "properties": { "PreviouslyVicePresident": true, "YearElected": 1801, - "age": "22", - "VicePresidents": ["Aaron Burr", "George Clinton"], - "description": "Sample description", - "name": "Test" + "VicePresidents": ["Aaron Burr", "George Clinton"] }, "profile": { "$email": "test@rudderstack.com", "$phone_number": "9112340375", - "$id": "sajal12" + "$id": "sajal12", + "age": "22", + "name": "Test", + "description": "Sample description" }, "value": 3000 } @@ -1152,12 +1158,12 @@ "properties": { "PreviouslyVicePresident": true, "YearElected": 1801, - "VicePresidents": ["Aaron Burr", "George Clinton"], - "age": "22" + "VicePresidents": ["Aaron Burr", "George Clinton"] }, "profile": { "$email": "test@rudderstack.com", "$phone_number": "9112340375", + "age": "22", "_id": "sajal12" } } @@ -1483,7 +1489,10 @@ "profile": { "$email": "test@rudderstack.com", "$phone_number": "9112340375", - "$id": "sajal12" + "$id": "sajal12", + "age": "22", + "name": "Test", + "description": "Sample description" }, "properties": { "ProductName": "test product", @@ -1493,10 +1502,7 @@ "URL": "http://www.example.com/path/to/product", "Brand": "Not for Kids", "Price": 9.9, - "Categories": ["Fiction", "Children"], - "age": "22", - "name": "Test", - "description": "Sample description" + "Categories": ["Fiction", "Children"] } } } @@ -1633,9 +1639,6 @@ "properties": { "$event_id": "1234", "$value": 20, - "age": "22", - "name": "Test", - "description": "Sample description", "items[0].ProductID": "123", "items[0].SKU": "G-32", "items[0].ProductName": "Monopoly", @@ -1652,7 +1655,10 @@ "profile": { "$email": "test@rudderstack.com", "$phone_number": "9112340375", - "$id": "sajal12" + "$id": "sajal12", + "age": "22", + "name": "Test", + "description": "Sample description" } } } @@ -1780,16 +1786,16 @@ "profile": { "$email": "test@rudderstack.com", "$phone_number": "9112340375", - "$id": "sajal12" + "$id": "sajal12", + "age": "22", + "name": "Test", + "description": "Sample description" }, "properties": { "$value": 12.12, "AddedItemCategories": ["Fiction3", "Children3"], "ItemNames": ["book1", "book2"], "CheckoutURL": "http://www.heythere.com", - "age": "22", - "name": "Test", - "description": "Sample description", "items": [ { "ProductID": "b1pid", From 02dcc850db65918a17f126bda9a360a460a76885 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 11 Sep 2023 07:10:42 +0000 Subject: [PATCH 5/5] chore(release): 1.41.0 --- CHANGELOG.md | 16 ++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 552026d41b..fdb53f4527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.41.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.40.2...v1.41.0) (2023-09-11) + + +### Features + +* add support for updating profile properties through track call ([#2581](https://github.com/rudderlabs/rudder-transformer/issues/2581)) ([f0f20d6](https://github.com/rudderlabs/rudder-transformer/commit/f0f20d654ec5ee8eb078ce7f5610a4666d73fd8c)) +* **INT-580:** messageId to event_id mapping support ([#2570](https://github.com/rudderlabs/rudder-transformer/issues/2570)) ([b38843b](https://github.com/rudderlabs/rudder-transformer/commit/b38843bce9bc02d73dceedc6f751f402251fd23a)) +* **tiktok_ads:** messageId to event_id mapping support ([72f87bf](https://github.com/rudderlabs/rudder-transformer/commit/72f87bfa381ed7a5b74fb5907f932b78d0257ab9)) + + +### Bug Fixes + +* **bugsnag:** alerts ([266514b](https://github.com/rudderlabs/rudder-transformer/commit/266514bd56c150d6c88c1db0733a1da0a4367c02)) +* **bugsnag:** alerts ([#2580](https://github.com/rudderlabs/rudder-transformer/issues/2580)) ([9e9eeac](https://github.com/rudderlabs/rudder-transformer/commit/9e9eeacdf79cf8175f87f302242542060f668db8)) +* json paths for non tracks events for warehouse ([#2571](https://github.com/rudderlabs/rudder-transformer/issues/2571)) ([e455368](https://github.com/rudderlabs/rudder-transformer/commit/e45536805cf9545b73f4d5bf1be5fad1565ab075)) + ### [1.40.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.40.1...v1.40.2) (2023-09-06) diff --git a/package-lock.json b/package-lock.json index 9372b3add9..8e478ec704 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.40.2", + "version": "1.41.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.40.2", + "version": "1.41.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "^0.7.24", diff --git a/package.json b/package.json index aecf5d581e..2241007ee8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.40.2", + "version": "1.41.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": {