diff --git a/CHANGELOG.md b/CHANGELOG.md index b62efe98d4..08d7a89e63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.46.4](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.3...v1.46.4) (2023-10-20) + + +### Bug Fixes + +* **hs:** time stamp and contact not found issue ([391c7cd](https://github.com/rudderlabs/rudder-transformer/commit/391c7cdc5576152576bbd2c6d335a04ce6795833)) + ### [1.46.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.2...v1.46.3) (2023-10-20) diff --git a/package-lock.json b/package-lock.json index f211d74b0f..991f958db0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.46.3", + "version": "1.46.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.46.3", + "version": "1.46.4", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "^0.7.24", diff --git a/package.json b/package.json index a920837643..503b1651ae 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.46.3", + "version": "1.46.4", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { diff --git a/src/v0/destinations/hs/HSTransform-v1.js b/src/v0/destinations/hs/HSTransform-v1.js index 387ecbf63f..5cb80f10f5 100644 --- a/src/v0/destinations/hs/HSTransform-v1.js +++ b/src/v0/destinations/hs/HSTransform-v1.js @@ -34,6 +34,7 @@ const { getEmailAndUpdatedProps, formatPropertyValueForIdentify, getHsSearchId, + populateTraits, } = require('./util'); const { JSON_MIME_TYPE } = require('../../util/constant'); @@ -52,7 +53,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); */ const processLegacyIdentify = async (message, destination, propertyMap) => { const { Config } = destination; - const traits = getFieldValueFromMessage(message, 'traits'); + let traits = getFieldValueFromMessage(message, 'traits'); const mappedToDestination = get(message, MappedToDestinationKey); const operation = get(message, 'context.hubspotOperation'); // if mappedToDestination is set true, then add externalId to traits @@ -80,6 +81,8 @@ const processLegacyIdentify = async (message, destination, propertyMap) => { )}/${hsSearchId}`; response.method = defaultPatchRequestConfig.requestMethod; } + + traits = await populateTraits(propertyMap, traits, destination); response.body.JSON = removeUndefinedAndNullValues({ properties: traits }); response.source = 'rETL'; response.operation = operation; diff --git a/src/v0/destinations/hs/HSTransform-v2.js b/src/v0/destinations/hs/HSTransform-v2.js index 75696b4e96..26c12d3eea 100644 --- a/src/v0/destinations/hs/HSTransform-v2.js +++ b/src/v0/destinations/hs/HSTransform-v2.js @@ -41,6 +41,7 @@ const { searchContacts, getEventAndPropertiesFromConfig, getHsSearchId, + populateTraits, } = require('./util'); const { JSON_MIME_TYPE } = require('../../util/constant'); @@ -69,7 +70,7 @@ const addHsAuthentication = (response, Config) => { */ const processIdentify = async (message, destination, propertyMap) => { const { Config } = destination; - const traits = getFieldValueFromMessage(message, 'traits'); + let traits = getFieldValueFromMessage(message, 'traits'); const mappedToDestination = get(message, MappedToDestinationKey); const operation = get(message, 'context.hubspotOperation'); const externalIdObj = getDestinationExternalIDObjectForRetl(message, 'HS'); @@ -124,6 +125,7 @@ const processIdentify = async (message, destination, propertyMap) => { response.method = defaultPatchRequestConfig.requestMethod; } + traits = await populateTraits(propertyMap, traits, destination); response.body.JSON = removeUndefinedAndNullValues({ properties: traits }); response.source = 'rETL'; response.operation = operation; diff --git a/src/v0/destinations/hs/transform.js b/src/v0/destinations/hs/transform.js index 3f8010ff49..a2326e0395 100644 --- a/src/v0/destinations/hs/transform.js +++ b/src/v0/destinations/hs/transform.js @@ -88,6 +88,7 @@ const processRouterDest = async (inputs, reqMetadata) => { if (mappedToDestination && GENERIC_TRUE_VALUES.includes(mappedToDestination?.toString())) { // skip splitting the batches to inserts and updates if object it is an association if (objectType.toLowerCase() !== 'association') { + propertyMap = await getProperties(destination); // get info about existing objects and splitting accordingly. tempInputs = await splitEventsForCreateUpdate(tempInputs, destination); } diff --git a/src/v0/destinations/hs/util.js b/src/v0/destinations/hs/util.js index ca92ed13cd..5d7a01da74 100644 --- a/src/v0/destinations/hs/util.js +++ b/src/v0/destinations/hs/util.js @@ -158,16 +158,14 @@ const validatePayloadDataTypes = (propertyMap, hsSupportedKey, value, traitsKey) if (propertyMap[hsSupportedKey] === 'bool' && typeof propValue === 'object') { throw new InstrumentationError( - `Property ${traitsKey} data type ${typeof propValue} is not matching with Hubspot property data type ${ - propertyMap[hsSupportedKey] + `Property ${traitsKey} data type ${typeof propValue} is not matching with Hubspot property data type ${propertyMap[hsSupportedKey] }`, ); } if (propertyMap[hsSupportedKey] === 'number' && typeof propValue !== 'number') { throw new InstrumentationError( - `Property ${traitsKey} data type ${typeof propValue} is not matching with Hubspot property data type ${ - propertyMap[hsSupportedKey] + `Property ${traitsKey} data type ${typeof propValue} is not matching with Hubspot property data type ${propertyMap[hsSupportedKey] }`, ); } @@ -175,6 +173,18 @@ const validatePayloadDataTypes = (propertyMap, hsSupportedKey, value, traitsKey) return propValue; }; +/** + * Converts date to UTC Midnight TimeStamp + * @param {*} propValue + * @returns + */ +const getUTCMidnightTimeStampValue = (propValue) => { + const time = propValue; + const date = new Date(time); + date.setUTCHours(0, 0, 0, 0); + return date.getTime(); +} + /** * add addtional properties in the payload that is provided in traits * only when it matches with HS properties (pre-defined/created from dashboard) @@ -204,10 +214,7 @@ const getTransformedJSON = async (message, destination, propertyMap) => { if (!rawPayload[traitsKey] && propertyMap[hsSupportedKey]) { let propValue = traits[traitsKey]; if (propertyMap[hsSupportedKey] === 'date') { - const time = propValue; - const date = new Date(time); - date.setUTCHours(0, 0, 0, 0); - propValue = date.getTime(); + propValue = getUTCMidnightTimeStampValue(propValue); } rawPayload[hsSupportedKey] = validatePayloadDataTypes( @@ -459,7 +466,7 @@ const getEventAndPropertiesFromConfig = (message, destination, payload) => { */ const getExistingData = async (inputs, destination) => { const { Config } = destination; - const values = []; + let values = []; let searchResponse; let updateHubspotIds = []; const firstMessage = inputs[0].message; @@ -478,8 +485,10 @@ const getExistingData = async (inputs, destination) => { inputs.map(async (input) => { const { message } = input; const { destinationExternalId } = getDestinationExternalIDInfoForRetl(message, DESTINATION); - values.push(destinationExternalId); + values.push(destinationExternalId.toString().toLowerCase()); }); + + values = Array.from(new Set(values)); const requestData = { filterGroups: [ { @@ -523,15 +532,15 @@ const getExistingData = async (inputs, destination) => { searchResponse = Config.authorizationType === 'newPrivateAppApi' ? await httpPOST(url, requestData, requestOptions, { - destType: 'hs', - feature: 'transformation', - endpointPath, - }) + destType: 'hs', + feature: 'transformation', + endpointPath, + }) : await httpPOST(url, requestData, { - destType: 'hs', - feature: 'transformation', - endpointPath, - }); + destType: 'hs', + feature: 'transformation', + endpointPath, + }); searchResponse = processAxiosResponse(searchResponse); if (searchResponse.status !== 200) { @@ -626,6 +635,31 @@ const getHsSearchId = (message) => { return { hsSearchId }; }; +/** + * returns updated traits + * @param {*} propertyMap + * @param {*} traits + * @param {*} destination + */ +const populateTraits = async (propertyMap, traits, destination) => { + const populatedTraits = traits; + let propertyToTypeMap = propertyMap; + if (!propertyToTypeMap) { + // fetch HS properties + propertyToTypeMap = await getProperties(destination); + } + + const keys = Object.keys(populatedTraits); + keys.forEach((key) => { + const value = populatedTraits[key]; + if (propertyToTypeMap[key] === 'date') { + populatedTraits[key] = getUTCMidnightTimeStampValue(value); + } + }) + + return populatedTraits; +} + module.exports = { validateDestinationConfig, formatKey, @@ -639,4 +673,6 @@ module.exports = { splitEventsForCreateUpdate, getHsSearchId, validatePayloadDataTypes, + getUTCMidnightTimeStampValue, + populateTraits, }; diff --git a/test/__mocks__/data/hs/response.json b/test/__mocks__/data/hs/response.json index 721ee64410..e552275466 100644 --- a/test/__mocks__/data/hs/response.json +++ b/test/__mocks__/data/hs/response.json @@ -3,6 +3,18 @@ { "name": "company_size", "type": "string" }, { "name": "date_of_birth", "type": "string" }, { "name": "days_to_close", "type": "number" }, + { + "name": "date_submitted", + "type": "date" + }, + { + "name": "days_create", + "type": "date" + }, + { + "name": "days_closed", + "type": "date" + }, { "name": "degree", "type": "string" }, { "name": "field_of_study", "type": "string" }, { "name": "first_conversion_date", "type": "datetime" }, @@ -192,6 +204,18 @@ { "name": "company_size", "type": "string" }, { "name": "date_of_birth", "type": "string" }, { "name": "days_to_close", "type": "number" }, + { + "name": "date_submitted", + "type": "date" + }, + { + "name": "date_created", + "type": "date" + }, + { + "name": "date_closed", + "type": "date" + }, { "name": "degree", "type": "string" }, { "name": "field_of_study", "type": "string" }, { "name": "first_conversion_date", "type": "datetime" }, diff --git a/test/__tests__/data/hs_router_rETL_input.json b/test/__tests__/data/hs_router_rETL_input.json index facb28ae82..3e855855b4 100644 --- a/test/__tests__/data/hs_router_rETL_input.json +++ b/test/__tests__/data/hs_router_rETL_input.json @@ -216,5 +216,117 @@ "metadata": { "jobId": 3 } + }, + { + "message": { + "channel": "web", + "context": { + "mappedToDestination": true, + "externalId": [ + { + "identifierType": "email", + "id": "testhubspotdatetime@email.com", + "type": "HS-lead" + } + ], + "sources": { + "job_id": "24c5HJxHomh6YCngEOCgjS5r1KX/Syncher", + "task_id": "vw_rs_mailchimp_mocked_hg_data", + "version": "v1.8.1", + "batch_id": "f252c69d-c40d-450e-bcd2-2cf26cb62762", + "job_run_id": "c8el40l6e87v0c4hkbl0", + "task_run_id": "c8el40l6e87v0c4hkblg" + } + }, + "type": "identify", + "traits": { + "firstname": "Test Hubspot", + "anonymousId": "123451", + "country": "India", + "date_submitted": "2023-09-25T17:31:04.128251Z", + "date_created": "2023-03-30T01:02:03.05Z", + "date_closed": "2023-10-18T04:38:59.229347Z" + }, + "messageId": "50360b9c-ea8d-409c-b672-c9230f91cce5", + "originalTimestamp": "2019-10-15T09:35:31.288Z", + "anonymousId": "00000000000000000000000000", + "userId": "12345", + "integrations": { + "All": true + }, + "sentAt": "2019-10-14T09:03:22.563Z" + }, + "destination": { + "Config": { + "authorizationType": "newPrivateAppApi", + "accessToken": "dummy-access-token", + "hubID": "dummy-hubId", + "apiKey": "dummy-apikey", + "apiVersion": "newApi", + "lookupField": "lookupField", + "hubspotEvents": [ + { + "rsEventName": "Purchase", + "hubspotEventName": "pedummy-hubId_rs_hub_test", + "eventProperties": [ + { + "from": "Revenue", + "to": "value" + }, + { + "from": "Price", + "to": "cost" + } + ] + }, + { + "rsEventName": "Order Complete", + "hubspotEventName": "pedummy-hubId_rs_hub_chair", + "eventProperties": [ + { + "from": "firstName", + "to": "first_name" + }, + { + "from": "lastName", + "to": "last_name" + } + ] + } + ], + "eventFilteringOption": "disable", + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "whitelistedEvents": [ + { + "eventName": "" + } + ] + }, + "secretConfig": {}, + "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", + "name": "Hubspot", + "enabled": true, + "workspaceId": "1TSN08muJTZwH8iCDmnnRt1pmLd", + "deleted": false, + "createdAt": "2020-12-30T08:39:32.005Z", + "updatedAt": "2021-02-03T16:22:31.374Z", + "destinationDefinition": { + "id": "1aIXqM806xAVm92nx07YwKbRrO9", + "name": "HS", + "displayName": "Hubspot", + "createdAt": "2020-04-09T09:24:31.794Z", + "updatedAt": "2021-01-11T11:03:28.103Z" + }, + "transformations": [], + "isConnectionEnabled": true, + "isProcessorEnabled": true + }, + "metadata": { + "jobId": 4 + } } ] diff --git a/test/__tests__/data/hs_router_rETL_output.json b/test/__tests__/data/hs_router_rETL_output.json index 5293a47cf9..dc7507220c 100644 --- a/test/__tests__/data/hs_router_rETL_output.json +++ b/test/__tests__/data/hs_router_rETL_output.json @@ -20,6 +20,17 @@ "country": "India 1", "email": "testhubspot@email.com" } + }, + { + "properties": { + "firstname": "Test Hubspot", + "anonymousId": "123451", + "country": "India", + "email": "testhubspotdatetime@email.com", + "date_closed": 1697587200000, + "date_created": 1680134400000, + "date_submitted": 1695600000000 + } } ] }, @@ -32,6 +43,9 @@ "metadata": [ { "jobId": 3 + }, + { + "jobId": 4 } ], "batched": true, @@ -214,4 +228,4 @@ "isProcessorEnabled": true } } -] +] \ No newline at end of file