diff --git a/package-lock.json b/package-lock.json index 40033e278b..14832566c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10621,7 +10621,9 @@ "license": "MIT" }, "node_modules/cross-spawn": { - "version": "7.0.3", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "license": "MIT", "dependencies": { "path-key": "^3.1.0", diff --git a/src/cdk/v2/destinations/pinterest_tag/utils.js b/src/cdk/v2/destinations/pinterest_tag/utils.js index 31a897f133..dbaeea9501 100644 --- a/src/cdk/v2/destinations/pinterest_tag/utils.js +++ b/src/cdk/v2/destinations/pinterest_tag/utils.js @@ -1,8 +1,8 @@ -/* eslint-disable no-param-reassign */ const sha256 = require('sha256'); const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const { API_VERSION } = require('./config'); +const { CommonUtils } = require('../../../../util/common'); const VALID_ACTION_SOURCES = ['app_android', 'app_ios', 'web', 'offline']; @@ -21,9 +21,15 @@ const ecomEventMaps = [ }, ]; -const USER_NON_ARRAY_PROPERTIES = ['client_user_agent', 'client_ip_address']; +const USER_NON_ARRAY_PROPERTIES = [ + 'client_user_agent', + 'client_ip_address', + 'click_id', + 'partner_id', +]; -const getHashedValue = (key, value) => { +const transformValue = (key, value) => { + const arrayValue = CommonUtils.toArray(value); switch (key) { case 'em': case 'ct': @@ -32,33 +38,18 @@ const getHashedValue = (key, value) => { case 'ln': case 'fn': case 'ge': - value = Array.isArray(value) - ? value.map((val) => val.toString().trim().toLowerCase()) - : value.toString().trim().toLowerCase(); - break; + return arrayValue.map((val) => val.toString().trim().toLowerCase()); case 'ph': - // phone numbers should only contain digits & should not contain leading zeros - value = Array.isArray(value) - ? value.map((val) => val.toString().replace(/\D/g, '').replace(/^0+/, '')) - : value.toString().replace(/\D/g, '').replace(/^0+/, ''); - break; + return arrayValue.map((val) => val.toString().replace(/\D/g, '').replace(/^0+/, '')); case 'zp': - // zip fields should only contain digits - value = Array.isArray(value) - ? value.map((val) => val.toString().trim().replace(/\D/g, '')) - : value.toString().replace(/\D/g, ''); - break; - case 'hashed_maids': - case 'external_id': - case 'db': - // no action needed on value - break; + return arrayValue.map((val) => val.toString().trim().replace(/\D/g, '')); default: - return String(value); + return arrayValue; } - return Array.isArray(value) ? value.map((val) => sha256(val)) : [sha256(value)]; }; +const getHashedValue = (key, value) => transformValue(key, value).map((val) => sha256(val)); + /** * * @param {*} userPayload Payload mapped from user fields @@ -67,10 +58,15 @@ const getHashedValue = (key, value) => { * Ref: https://s.pinimg.com/ct/docs/conversions_api/dist/v3.html */ const processUserPayload = (userPayload) => { - Object.keys(userPayload).forEach((key) => { - userPayload[key] = getHashedValue(key, userPayload[key]); + const newPayload = { ...userPayload }; + Object.keys(newPayload).forEach((key) => { + if (USER_NON_ARRAY_PROPERTIES.includes(key)) { + newPayload[key] = String(newPayload[key]); + } else { + newPayload[key] = getHashedValue(key, newPayload[key]); + } }); - return userPayload; + return newPayload; }; /** @@ -99,11 +95,7 @@ const processHashedUserPayload = (userPayload, message) => { const processedHashedUserPayload = {}; Object.keys(userPayload).forEach((key) => { if (!USER_NON_ARRAY_PROPERTIES.includes(key)) { - if (Array.isArray(userPayload[key])) { - processedHashedUserPayload[key] = [...userPayload[key]]; - } else { - processedHashedUserPayload[key] = [userPayload[key]]; - } + processedHashedUserPayload[key] = CommonUtils.toArray(userPayload[key]); } else { processedHashedUserPayload[key] = userPayload[key]; } @@ -111,10 +103,8 @@ const processHashedUserPayload = (userPayload, message) => { // multiKeyMap will works on only specific values like m, male, MALE, f, F, Female // if hashed data is sent from the user, it is directly set over here const gender = message.traits?.gender || message.context?.traits?.gender; - if (gender && Array.isArray(gender)) { - processedHashedUserPayload.ge = [...gender]; - } else if (gender) { - processedHashedUserPayload.ge = [gender]; + if (gender) { + processedHashedUserPayload.ge = CommonUtils.toArray(gender); } return processedHashedUserPayload; }; diff --git a/src/v0/destinations/amazon_audience/utils.js b/src/v0/destinations/amazon_audience/utils.js index c25f301378..350d071a47 100644 --- a/src/v0/destinations/amazon_audience/utils.js +++ b/src/v0/destinations/amazon_audience/utils.js @@ -21,7 +21,8 @@ const buildResponseWithUsers = (users, action, config, jobIdList, secret) => { if (!secret?.clientId) { throw new OAuthSecretError('OAuth - Client Id not found'); } - const externalId = `Rudderstack_${sha256(`${jobIdList}`)}`; + const jobIdHash = sha256(String(jobIdList)); + const externalId = `Rudderstack_${jobIdHash}`; const response = defaultRequestConfig(); response.endpoint = ''; response.method = defaultPostRequestConfig.requestMethod; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js index 5866b66538..c039ff1486 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js @@ -1,4 +1,3 @@ -/* eslint-disable no-const-assign */ const lodash = require('lodash'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); const { @@ -19,29 +18,23 @@ const { const { getErrorResponse, createFinalResponse } = require('../../util/recordUtils'); const { offlineDataJobsMapping, consentConfigMap } = require('./config'); -const processRecordEventArray = ( - records, - message, - destination, - accessToken, - developerToken, - audienceId, - typeOfList, - userSchema, - isHashRequired, - userDataConsent, - personalizationConsent, - operationType, -) => { - let outputPayloads = {}; - // ** only send it if identifier > 0 - - const fieldsArray = []; - const metadata = []; - records.forEach((record) => { - fieldsArray.push(record.message.fields); - metadata.push(record.metadata); - }); +const processRecordEventArray = (records, context, operationType) => { + const { + message, + destination, + accessToken, + developerToken, + audienceId, + typeOfList, + userSchema, + isHashRequired, + userDataConsent, + personalizationConsent, + } = context; + + const outputPayloads = {}; + const fieldsArray = records.map((record) => record.message.fields); + const metadata = records.map((record) => record.metadata); const userIdentifiersList = populateIdentifiersForRecordEvent( fieldsArray, @@ -52,130 +45,65 @@ const processRecordEventArray = ( const outputPayload = constructPayload(message, offlineDataJobsMapping); outputPayload.operations = []; - // breaking the userIdentiFier array in chunks of 20 - const userIdentifierChunks = returnArrayOfSubarrays(userIdentifiersList, 20); - // putting each chunk in different create/remove operations - switch (operationType) { - case 'add': - // for add operation - userIdentifierChunks.forEach((element) => { - const operations = { - create: {}, - }; - operations.create.userIdentifiers = element; - outputPayload.operations.push(operations); - }); - outputPayloads = { ...outputPayloads, create: outputPayload }; - break; - case 'remove': - // for remove operation - userIdentifierChunks.forEach((element) => { - const operations = { - remove: {}, - }; - operations.remove.userIdentifiers = element; - outputPayload.operations.push(operations); - }); - outputPayloads = { ...outputPayloads, remove: outputPayload }; - break; - default: - } - const toSendEvents = []; - Object.values(outputPayloads).forEach((data) => { - const consentObj = populateConsentFromConfig( - { userDataConsent, personalizationConsent }, - consentConfigMap, - ); - toSendEvents.push( - responseBuilder(accessToken, developerToken, data, destination, audienceId, consentObj), - ); + const userIdentifierChunks = returnArrayOfSubarrays(userIdentifiersList, 20); + userIdentifierChunks.forEach((chunk) => { + const operation = { + [operationType]: { userIdentifiers: chunk }, + }; + outputPayload.operations.push(operation); }); - const successResponse = getSuccessRespEvents(toSendEvents, metadata, destination, true); + outputPayloads[operationType] = outputPayload; + + const consentObj = populateConsentFromConfig( + { userDataConsent, personalizationConsent }, + consentConfigMap, + ); + + const toSendEvents = Object.values(outputPayloads).map((data) => + responseBuilder(accessToken, developerToken, data, destination, audienceId, consentObj), + ); - return successResponse; + return getSuccessRespEvents(toSendEvents, metadata, destination, true); }; -function preparepayload(events, config) { +function preparePayload(events, config) { const { destination, message, metadata } = events[0]; const accessToken = getAccessToken(metadata, 'access_token'); const developerToken = getValueFromMessage(metadata, 'secret.developer_token'); - const { - audienceId, - typeOfList, - isHashRequired, - userSchema, - userDataConsent, - personalizationConsent, - } = config; + + const context = { + message, + destination, + accessToken, + developerToken, + ...config, + }; const groupedRecordsByAction = lodash.groupBy(events, (record) => record.message.action?.toLowerCase(), ); - let insertResponse; - let deleteResponse; - let updateResponse; - - if (groupedRecordsByAction.delete) { - deleteResponse = processRecordEventArray( - groupedRecordsByAction.delete, - message, - destination, - accessToken, - developerToken, - audienceId, - typeOfList, - userSchema, - isHashRequired, - userDataConsent, - personalizationConsent, - 'remove', - ); - } - - if (groupedRecordsByAction.insert) { - insertResponse = processRecordEventArray( - groupedRecordsByAction.insert, - message, - destination, - accessToken, - developerToken, - audienceId, - typeOfList, - userSchema, - isHashRequired, - userDataConsent, - personalizationConsent, - 'add', - ); - } - - if (groupedRecordsByAction.update) { - updateResponse = processRecordEventArray( - groupedRecordsByAction.update, - message, - destination, - accessToken, - developerToken, - audienceId, - typeOfList, - userSchema, - isHashRequired, - userDataConsent, - personalizationConsent, - 'add', - ); - } + const actionResponses = ['delete', 'insert', 'update'].reduce((responses, action) => { + const operationType = action === 'delete' ? 'remove' : 'create'; + if (groupedRecordsByAction[action]) { + return { + ...responses, + [action]: processRecordEventArray(groupedRecordsByAction[action], context, operationType), + }; + } + return responses; + }, {}); const errorResponse = getErrorResponse(groupedRecordsByAction); const finalResponse = createFinalResponse( - deleteResponse, - insertResponse, - updateResponse, + actionResponses.delete, + actionResponses.insert, + actionResponses.update, errorResponse, ); + if (finalResponse.length === 0) { throw new InstrumentationError( 'Missing valid parameters, unable to generate transformed payload', @@ -196,14 +124,16 @@ function processRecordInputsV0(groupedRecordInputs) { personalizationConsent, } = destination.Config; - return preparepayload(groupedRecordInputs, { + const config = { audienceId: getOperationAudienceId(audienceId, message), typeOfList, userSchema, isHashRequired, userDataConsent, personalizationConsent, - }); + }; + + return preparePayload(groupedRecordInputs, config); } function processRecordInputsV1(groupedRecordInputs) { @@ -211,11 +141,7 @@ function processRecordInputsV1(groupedRecordInputs) { const { audienceId, typeOfList, isHashRequired, userDataConsent, personalizationConsent } = connection.config.destination; - const identifiers = message?.identifiers; - let userSchema; - if (identifiers) { - userSchema = Object.keys(identifiers); - } + const userSchema = message?.identifiers ? Object.keys(message.identifiers) : undefined; const events = groupedRecordInputs.map((record) => ({ ...record, @@ -225,23 +151,23 @@ function processRecordInputsV1(groupedRecordInputs) { }, })); - return preparepayload(events, { + const config = { audienceId, typeOfList, userSchema, isHashRequired, userDataConsent, personalizationConsent, - }); + }; + + return preparePayload(events, config); } function processRecordInputs(groupedRecordInputs) { const event = groupedRecordInputs[0]; - // First check for rETL flow and second check for ES flow - if (isEventSentByVDMV1Flow(event) || !isEventSentByVDMV2Flow(event)) { - return processRecordInputsV0(groupedRecordInputs); - } - return processRecordInputsV1(groupedRecordInputs); + return isEventSentByVDMV1Flow(event) || !isEventSentByVDMV2Flow(event) + ? processRecordInputsV0(groupedRecordInputs) + : processRecordInputsV1(groupedRecordInputs); } module.exports = {