Skip to content

Commit

Permalink
chore(release): pull main into develop post release v1.79.1 (#3758)
Browse files Browse the repository at this point in the history
  • Loading branch information
krishna2020 authored Sep 24, 2024
2 parents d40e772 + 4c0c681 commit 4b54592
Show file tree
Hide file tree
Showing 33 changed files with 2,501 additions and 184 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@

All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.

### [1.79.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.79.0...v1.79.1) (2024-09-24)


### Bug Fixes

* allow users context traits and underscore divide numbers configuration ([#3752](https://github.com/rudderlabs/rudder-transformer/issues/3752)) ([386d2ab](https://github.com/rudderlabs/rudder-transformer/commit/386d2ab88c0fe72dc47ba119be08ad1c0cd6d51b))
* populate users fields for sentAt, timestamp and originalTimestamp ([#3753](https://github.com/rudderlabs/rudder-transformer/issues/3753)) ([f50effe](https://github.com/rudderlabs/rudder-transformer/commit/f50effeeabdb888f82451c225a80971dbe6532b6))
* prefer event check vs config check for vdm ([#3754](https://github.com/rudderlabs/rudder-transformer/issues/3754)) ([b2c1a18](https://github.com/rudderlabs/rudder-transformer/commit/b2c1a1893dfb957ac7a24c000b33cd254ef54b6c))
* support different lookup fields and custom_attributes for rETL events ([#3751](https://github.com/rudderlabs/rudder-transformer/issues/3751)) ([10d914e](https://github.com/rudderlabs/rudder-transformer/commit/10d914e25203bd6ae95801c2a98c17690bd2d6ef))

## [1.79.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.78.0...v1.79.0) (2024-09-20)


Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "rudder-transformer",
"version": "1.79.0",
"version": "1.79.1",
"description": "",
"homepage": "https://github.com/rudderlabs/rudder-transformer#readme",
"bugs": {
Expand Down
10 changes: 5 additions & 5 deletions src/cdk/v2/destinations/intercom/procWorkflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ steps:
template: |
const payload = .message.context.mappedToDestination ? $.outputs.rEtlPayload : $.outputs.identifyTransformationForLatestVersion;
payload.name = $.getName(.message);
payload.custom_attributes = .message.context.traits || {};
payload.custom_attributes = $.filterCustomAttributes(payload, "user", .destination);
payload.custom_attributes = (.message.context.mappedToDestination ? .message.traits.custom_attributes : .message.context.traits) || {};
payload.custom_attributes = $.filterCustomAttributes(payload, "user", .destination, .message);
payload.external_id = !payload.external_id && .destination.Config.sendAnonymousId && .message.anonymousId ? .message.anonymousId : payload.external_id;
$.context.payload = payload;
$.assert($.context.payload.external_id || $.context.payload.email, "Either email or userId is required for Identify call");
Expand Down Expand Up @@ -114,7 +114,7 @@ steps:
update_last_request_at: typeof .destination.Config.updateLastRequestAt === 'boolean' ? .destination.Config.updateLastRequestAt : true
}
payload.companies = $.getCompaniesList(payload);
payload.custom_attributes = !.message.context.mappedToDestination ? $.filterCustomAttributes(payload, "user", .destination);
payload.custom_attributes = !.message.context.mappedToDestination ? $.filterCustomAttributes(payload, "user", .destination,.message);
payload.user_id = !payload.user_id && .destination.Config.sendAnonymousId && .message.anonymousId ? .message.anonymousId : payload.user_id;
$.context.payload = payload;
$.assert($.context.payload.user_id || $.context.payload.email, "Either of `email` or `userId` is required for Identify call");
Expand Down Expand Up @@ -175,7 +175,7 @@ steps:
$.assert(.message.groupId, "groupId is required for group call");
const payload = .message.context.mappedToDestination ? $.outputs.rEtlPayload : $.outputs.groupTransformation;
payload.custom_attributes = .message.traits || {};
payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination);
payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination,.message);
$.context.payload = payload;
- name: whenSearchContactFound
condition: $.isDefinedAndNotNull($.outputs.searchContact)
Expand Down Expand Up @@ -214,7 +214,7 @@ steps:
...payload,
custom_attributes : $.getFieldValueFromMessage(.message, "traits") || {}
}
payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination);
payload.custom_attributes = $.filterCustomAttributes(payload, "company", .destination, .message);
response.body.JSON = $.removeUndefinedAndNullValues(payload);
response.endpoint = $.getBaseEndpoint(.destination) + "/" + "companies";
response.headers = $.getHeaders(.destination, $.outputs.apiVersion);
Expand Down
25 changes: 17 additions & 8 deletions src/cdk/v2/destinations/intercom/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -233,20 +233,26 @@ const attachUserAndCompany = (message, Config) => {
* @param {*} type
* @returns
*/
const filterCustomAttributes = (payload, type, destination) => {
const filterCustomAttributes = (payload, type, destination, message) => {
let ReservedAttributesList;
let { apiVersion } = destination.Config;
apiVersion = isDefinedAndNotNull(apiVersion) ? apiVersion : 'v2';
// we are discarding the lookup field from custom attributes
const lookupField = getLookUpField(message);
if (type === 'user') {
ReservedAttributesList =
apiVersion === 'v1'
ReservedAttributesList = [
...(apiVersion === 'v1'
? ReservedAttributes.v1UserAttributes
: ReservedAttributes.v2UserAttributes;
: ReservedAttributes.v2UserAttributes),
lookupField,
];
} else {
ReservedAttributesList =
apiVersion === 'v1'
ReservedAttributesList = [
...(apiVersion === 'v1'
? ReservedAttributes.v1CompanyAttributes
: ReservedAttributes.v2CompanyAttributes;
: ReservedAttributes.v2CompanyAttributes),
lookupField !== 'email' && lookupField,
];
}
let customAttributes = { ...get(payload, 'custom_attributes') };
if (customAttributes) {
Expand All @@ -270,7 +276,10 @@ const filterCustomAttributes = (payload, type, destination) => {
*/
const searchContact = async (message, destination, metadata) => {
const lookupField = getLookUpField(message);
const lookupFieldValue = getFieldValueFromMessage(message, lookupField);
let lookupFieldValue = getFieldValueFromMessage(message, lookupField);
if (!lookupFieldValue) {
lookupFieldValue = message?.context?.traits?.[lookupField];
}
const data = JSON.stringify({
query: {
operator: 'AND',
Expand Down
7 changes: 4 additions & 3 deletions src/v0/destinations/fb_custom_audience/recordTransform.js
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,11 @@ const processRecordInputsV2 = (groupedRecordInputs) => {

function processRecordInputs(groupedRecordInputs) {
const event = groupedRecordInputs[0];
if (isEventSentByVDMV2Flow(event)) {
return processRecordInputsV2(groupedRecordInputs);
// First check for rETL flow and second check for ES flow
if (isEventSentByVDMV1Flow(event) || !isEventSentByVDMV2Flow(event)) {
return processRecordInputsV1(groupedRecordInputs);
}
return processRecordInputsV1(groupedRecordInputs);
return processRecordInputsV2(groupedRecordInputs);
}

module.exports = {
Expand Down
48 changes: 37 additions & 11 deletions src/warehouse/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,18 @@ function processWarehouseMessage(message, options) {
const skipReservedKeywordsEscaping =
options.integrationOptions.skipReservedKeywordsEscaping || false;

// underscoreDivideNumbers when set to false, if a column has a format like "_v_3_", it will be formatted to "_v3_"
// underscoreDivideNumbers when set to true, if a column has a format like "_v_3_", we keep it like that
// For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false.
options.underscoreDivideNumbers = options.destConfig?.underscoreDivideNumbers || false;

// allowUsersContextTraits when set to true, if context.traits.* is present, it will be added as context_traits_* and *,
// e.g., for context.traits.name, context_traits_name and name will be added to the user's table.
// allowUsersContextTraits when set to false, if context.traits.* is present, it will be added only as context_traits_*
// e.g., for context.traits.name, only context_traits_name will be added to the user's table.
// For older destinations, it will come as true, and for new destinations this config will not be present, which means we will treat it as false.
const allowUsersContextTraits = options.destConfig?.allowUsersContextTraits || false;

addJsonKeysToOptions(options);

if (isBlank(message.messageId)) {
Expand Down Expand Up @@ -898,16 +910,18 @@ function processWarehouseMessage(message, options) {
`${eventType + '_userProperties_'}`,
2,
);
setDataFromInputAndComputeColumnTypes(
utils,
eventType,
commonProps,
message.context ? message.context.traits : {},
commonColumnTypes,
options,
`${eventType + '_context_traits_'}`,
3,
);
if (allowUsersContextTraits) {
setDataFromInputAndComputeColumnTypes(
utils,
eventType,
commonProps,
message.context ? message.context.traits : {},
commonColumnTypes,
options,
`${eventType + '_context_traits_'}`,
3,
);
}
setDataFromInputAndComputeColumnTypes(
utils,
eventType,
Expand Down Expand Up @@ -987,11 +1001,23 @@ function processWarehouseMessage(message, options) {

const usersEvent = { ...commonProps };
const usersColumnTypes = {};
let userColumnMappingRules = whUserColumnMappingRules;
if (!isDataLakeProvider(options.provider)) {
userColumnMappingRules = {
...userColumnMappingRules,
...{
sent_at: 'sentAt',
timestamp: 'timestamp',
original_timestamp: 'originalTimestamp',
},
};
}

setDataFromColumnMappingAndComputeColumnTypes(
utils,
usersEvent,
message,
whUserColumnMappingRules,
userColumnMappingRules,
usersColumnTypes,
options,
);
Expand Down
37 changes: 37 additions & 0 deletions src/warehouse/snakecase/snakecase.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
const { toString } = require('lodash');
const { unicodeWords, unicodeWordsWithNumbers } = require('./unicodeWords');

const hasUnicodeWord = RegExp.prototype.test.bind(
/[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/,
);

/** Used to match words composed of alphanumeric characters. */
// eslint-disable-next-line no-control-regex
const reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g;

function asciiWords(string) {
return string.match(reAsciiWord);
}

function words(string) {
const result = hasUnicodeWord(string) ? unicodeWords(string) : asciiWords(string);
return result || [];
}

function wordsWithNumbers(string) {
const result = hasUnicodeWord(string) ? unicodeWordsWithNumbers(string) : asciiWords(string);
return result || [];
}

const snakeCase = (string) =>
words(toString(string).replace(/['\u2019]/g, '')).reduce(
(result, word, index) => result + (index ? '_' : '') + word.toLowerCase(),
'',
);
const snakeCaseWithNumbers = (string) =>
wordsWithNumbers(toString(string).replace(/['\u2019]/g, '')).reduce(
(result, word, index) => result + (index ? '_' : '') + word.toLowerCase(),
'',
);

module.exports = { words, wordsWithNumbers, snakeCase, snakeCaseWithNumbers };
94 changes: 94 additions & 0 deletions src/warehouse/snakecase/unicodeWords.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/** Used to compose unicode character classes. */
const rsAstralRange = '\\ud800-\\udfff';
const rsComboMarksRange = '\\u0300-\\u036f';
const reComboHalfMarksRange = '\\ufe20-\\ufe2f';
const rsComboSymbolsRange = '\\u20d0-\\u20ff';
const rsComboMarksExtendedRange = '\\u1ab0-\\u1aff';
const rsComboMarksSupplementRange = '\\u1dc0-\\u1dff';
const rsComboRange =
rsComboMarksRange +
reComboHalfMarksRange +
rsComboSymbolsRange +
rsComboMarksExtendedRange +
rsComboMarksSupplementRange;
const rsDingbatRange = '\\u2700-\\u27bf';
const rsLowerRange = 'a-z\\xdf-\\xf6\\xf8-\\xff';
const rsMathOpRange = '\\xac\\xb1\\xd7\\xf7';
const rsNonCharRange = '\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf';
const rsPunctuationRange = '\\u2000-\\u206f';
const rsSpaceRange =
' \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000';
const rsUpperRange = 'A-Z\\xc0-\\xd6\\xd8-\\xde';
const rsVarRange = '\\ufe0e\\ufe0f';
const rsBreakRange = rsMathOpRange + rsNonCharRange + rsPunctuationRange + rsSpaceRange;

/** Used to compose unicode capture groups. */
const rsApos = "['\u2019]";
const rsBreak = `[${rsBreakRange}]`;
const rsCombo = `[${rsComboRange}]`;
const rsDigit = '\\d';
const rsDingbat = `[${rsDingbatRange}]`;
const rsLower = `[${rsLowerRange}]`;
const rsMisc = `[^${rsAstralRange}${rsBreakRange + rsDigit + rsDingbatRange + rsLowerRange + rsUpperRange}]`;
const rsFitz = '\\ud83c[\\udffb-\\udfff]';
const rsModifier = `(?:${rsCombo}|${rsFitz})`;
const rsNonAstral = `[^${rsAstralRange}]`;
const rsRegional = '(?:\\ud83c[\\udde6-\\uddff]){2}';
const rsSurrPair = '[\\ud800-\\udbff][\\udc00-\\udfff]';
const rsUpper = `[${rsUpperRange}]`;
const rsZWJ = '\\u200d';

/** Used to compose unicode regexes. */
const rsMiscLower = `(?:${rsLower}|${rsMisc})`;
const rsMiscUpper = `(?:${rsUpper}|${rsMisc})`;
const rsOptContrLower = `(?:${rsApos}(?:d|ll|m|re|s|t|ve))?`;
const rsOptContrUpper = `(?:${rsApos}(?:D|LL|M|RE|S|T|VE))?`;
const reOptMod = `${rsModifier}?`;
const rsOptVar = `[${rsVarRange}]?`;
const rsOptJoin = `(?:${rsZWJ}(?:${[rsNonAstral, rsRegional, rsSurrPair].join('|')})${rsOptVar + reOptMod})*`;
const rsOrdLower = '\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])';
const rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])';
const rsSeq = rsOptVar + reOptMod + rsOptJoin;
const rsEmoji = `(?:${[rsDingbat, rsRegional, rsSurrPair].join('|')})${rsSeq}`;

const reUnicodeWords = RegExp(
[
`${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions
`${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions
`${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions
`${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS")
rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND")
rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd")
`${rsDigit}+`, // Pure digits (e.g., "123")
rsEmoji, // Emojis (e.g., 😀, ❤️)
].join('|'),
'g',
);

const reUnicodeWordsWithNumbers = RegExp(
[
`${rsUpper}?${rsLower}+${rsDigit}+`, // Lowercase letters followed by digits (e.g., "abc123")
`${rsUpper}+${rsDigit}+`, // Uppercase letters followed by digits (e.g., "ABC123")
`${rsDigit}+${rsUpper}?${rsLower}+`, // Digits followed by lowercase letters (e.g., "123abc")
`${rsDigit}+${rsUpper}+`, // Digits followed by uppercase letters (e.g., "123ABC")
`${rsUpper}?${rsLower}+${rsOptContrLower}(?=${[rsBreak, rsUpper, '$'].join('|')})`, // Regular words, lowercase letters followed by optional contractions
`${rsMiscUpper}+${rsOptContrUpper}(?=${[rsBreak, rsUpper + rsMiscLower, '$'].join('|')})`, // Miscellaneous uppercase characters with optional contractions
`${rsUpper}?${rsMiscLower}+${rsOptContrLower}`, // Miscellaneous lowercase sequences with optional contractions
`${rsUpper}+${rsOptContrUpper}`, // All uppercase words with optional contractions (e.g., "THIS")
rsOrdUpper, // Ordinals for uppercase (e.g., "1ST", "2ND")
rsOrdLower, // Ordinals for lowercase (e.g., "1st", "2nd")
`${rsDigit}+`, // Pure digits (e.g., "123")
rsEmoji, // Emojis (e.g., 😀, ❤️)
].join('|'),
'g',
);

function unicodeWords(string) {
return string.match(reUnicodeWords);
}

function unicodeWordsWithNumbers(string) {
return string.match(reUnicodeWordsWithNumbers);
}

module.exports = { unicodeWords, unicodeWordsWithNumbers };
11 changes: 1 addition & 10 deletions src/warehouse/util.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
const _ = require('lodash');
const get = require('get-value');

const v0 = require('./v0/util');
const v1 = require('./v1/util');
const { PlatformError, InstrumentationError } = require('@rudderstack/integrations-lib');
const { isBlank } = require('./config/helpers');
Expand Down Expand Up @@ -112,14 +110,7 @@ function validTimestamp(input) {
}

function getVersionedUtils(schemaVersion) {
switch (schemaVersion) {
case 'v0':
return v0;
case 'v1':
return v1;
default:
return v1;
}
return v1;
}

function isRudderSourcesEvent(event) {
Expand Down
Loading

0 comments on commit 4b54592

Please sign in to comment.