chore: add logger at integration level

package.json

@@ -65,8 +65,8 @@
     "@ndhoule/extend": "^2.0.0",
     "@pyroscope/nodejs": "^0.2.9",
     "@rudderstack/integrations-lib": "^0.2.10",
-    "@rudderstack/json-template-engine": "^0.13.0",
-    "@rudderstack/workflow-engine": "^0.8.1",
+    "@rudderstack/json-template-engine": "^0.13.2",
+    "@rudderstack/workflow-engine": "^0.8.2",
     "@shopify/jest-koa-mocks": "^5.1.1",
     "ajv": "^8.12.0",
     "ajv-draft-04": "^1.0.0",

src/adapters/network.js

@@ -94,7 +94,7 @@ const fireHTTPStats = (clientResponse, startTime, statTags) => {
     startTime,
     clientResponse,
   };
-  if (statTags?.metadata && !Array.isArray(statTags?.metadata)) {
+  if (statTags?.metadata) {
     const metadata = !Array.isArray(statTags?.metadata) ? [statTags.metadata] : statTags.metadata;
     metadata?.forEach((m) => {
       fireOutgoingReqStats({

src/cdk/v2/destinations/gladly/procWorkflow.yaml

@@ -61,7 +61,8 @@ steps:
         headers: $.getHeaders(.destination)
       }
       const endpoint = $.getEndpoint(.destination) + "?" + $.getQueryParams($.context.payload);
-      const rawResponse = await $.httpGET(endpoint,requestOptions)
+      const reqStats = {metadata:.metadata, module: 'router',feature: "transformation", destType:"gladly",requestMethod:"get",endpointPath:"/api/v1/customer-profiles"}
+      const rawResponse = await $.httpGET(endpoint,requestOptions, reqStats)
       const processedResponse = $.processAxiosResponse(rawResponse)
       processedResponse
 

src/cdk/v2/handler.ts

@@ -88,7 +88,8 @@ export async function processCdkV2Workflow(
   bindings: Record<string, FixMe> = {},
 ) {
   try {
-    logger.debug(`Processing cdkV2 workflow`);
+    logger.debug(`Processing cdkV2 workflow`, { destType });
+
     const workflowEngine = await getCachedWorkflowEngine(destType, feature, bindings);
     return await executeWorkflow(workflowEngine, parsedEvent, requestMetadata);
   } catch (error) {

src/index.ts

@@ -35,7 +35,7 @@ app.use(
 addRequestSizeMiddleware(app);
 addSwaggerRoutes(app);
 
-logger.error('Using new routes');
+logger.info('Using new routes');
 applicationRoutes(app);
 
 function finalFunction() {

src/logger.js

@@ -1,17 +1,32 @@
-const dotenv = require('dotenv');
-
 /* istanbul ignore file */
 const { LOGLEVELS, structuredLogger } = require('@rudderstack/integrations-lib');
 
-// any value greater than levelError will work as levelNone
+// LOGGER_IMPL can be `console` or `winston`
 const loggerImpl = process.env.LOGGER_IMPL ?? 'winston';
 
 let logLevel = process.env.LOG_LEVEL ?? 'error';
 
-const logger = structuredLogger({ level: logLevel });
+const logger = structuredLogger({
+  level: logLevel,
+  fillExcept: [
+    'destinationId',
+    'sourceId',
+    'destinationType',
+    'workspaceId',
+    'module',
+    'implementation',
+    'feature',
+    'destType',
+  ],
+});
 
 const getLogger = () => {
-  return loggerImpl === 'winston' ? logger : console;
+  switch (loggerImpl) {
+    case 'winston':
+      return logger;
+    case 'console':
+      return console;
+  }
 };
 
 const setLogLevel = (level) => {
@@ -31,19 +46,45 @@ const getLogMetadata = (metadata) => {
   if (Array.isArray(metadata)) {
     [reqMeta] = metadata;
   }
+  const destType = reqMeta?.destType || reqMeta?.destinationType;
   return {
     ...(reqMeta?.destinationId && { destinationId: reqMeta.destinationId }),
     ...(reqMeta?.sourceId && { sourceId: reqMeta.sourceId }),
     ...(reqMeta?.workspaceId && { workspaceId: reqMeta.workspaceId }),
-    ...(reqMeta?.destType && { destType: reqMeta.destType }),
+    ...(destType && { destType }),
     ...(reqMeta?.module && { module: reqMeta.module }),
     ...(reqMeta?.implementation && { implementation: reqMeta.implementation }),
     ...(reqMeta?.feature && { feature: reqMeta.feature }),
   };
 };
 
-const log = (logMethod, args) => {
-  const [message, logInfo, ...otherArgs] = args;
+const formLogArgs = (args) => {
+  let msg = '';
+  let otherArgs = [];
+  args.forEach((arg) => {
+    if (typeof arg !== 'object') {
+      msg += ' ' + arg;
+      return;
+    }
+    otherArgs.push(arg);
+  });
+  return [msg, ...otherArgs];
+};
+
+/**
+ * Perform logging operation on logMethod passed
+ *
+ * **Good practices**:
+ * - Do not have more than one array args in logger
+ * @param {*} logMethod
+ *  - instance method reference
+ *  - The logger should implement all of debug/info/warn/error methods
+ * @param {*} logArgs
+ *  - the arguments that needs to be passed to logger instance method
+ */
+const log = (logMethod, logArgs) => {
+  const [message, ...args] = formLogArgs(logArgs);
+  const [logInfo, ...otherArgs] = args;
   if (logInfo) {
     const { metadata, ...otherLogInfoArgs } = logInfo;
     if (Array.isArray(metadata)) {
@@ -74,7 +115,7 @@ const log = (logMethod, args) => {
 
 const debug = (...args) => {
   const logger = getLogger();
-  if (LOGLEVELS.debug <= logLevel) {
+  if (LOGLEVELS.debug <= LOGLEVELS[logLevel]) {
     log(logger.debug, args);
   }
 };

src/services/destination/postTransformation.ts

@@ -19,7 +19,7 @@
 import { generateErrorObject } from '../../v0/util';
 import tags from '../../v0/util/tags';
 import { ErrorReportingService } from '../errorReporting';
-import { MiscService } from '../misc';
+import logger from '../../logger';
 
 const defaultErrorMessages = {
   router: '[Router Transform] Error occurred while processing the payload.',
@@ -68,7 +68,7 @@
       error: errObj.message || '[Processor Transform] Error occurred while processing the payload.',
       statTags: errObj.statTags,
     } as ProcessorTransformationResponse;
-    MiscService.logError(
+    logger.error(
       errObj.message || '[Processor Transform] Error occurred while processing the payload.',
       metaTo.errorDetails,
     );
@@ -109,7 +109,7 @@
           ...resp.statTags,
           ...metaTo.errorDetails,
         };
-        MiscService.logError(resp.error || defaultErrorMessages.router, metaTo.errorDetails);
+        logger.error(resp.error || defaultErrorMessages.router, metaTo.errorDetails);
         stats.increment('event_transform_failure', metaTo.errorDetails);
       } else {
         stats.increment('event_transform_success', {
@@ -138,7 +138,7 @@
       error: errObj.message || defaultErrorMessages.router,
       statTags: errObj.statTags,
     } as RouterTransformationResponse;
-    MiscService.logError(errObj.message || defaultErrorMessages.router, metaTo.errorDetails);
+    logger.error(errObj.message || defaultErrorMessages.router, metaTo.errorDetails);
     ErrorReportingService.reportError(error, metaTo.errorContext, resp);
     stats.increment('event_transform_failure', metaTo.errorDetails);
     return resp;
@@ -156,7 +156,7 @@
       error: errObj.message || defaultErrorMessages.delivery,
       statTags: errObj.statTags,
     } as RouterTransformationResponse;
-    MiscService.logError(error as string, metaTo.errorDetails);
+    logger.error(error as string, metaTo.errorDetails);
     ErrorReportingService.reportError(error, metaTo.errorContext, resp);
     return resp;
   }
@@ -187,10 +187,7 @@
     const errObj = generateErrorObject(error, metaTo.errorDetails, false);
     const metadataArray = metaTo.metadatas;
     if (!Array.isArray(metadataArray)) {
-      MiscService.logError(
-        'Proxy v1 endpoint error : metadataArray is not an array',
-        metaTo.errorDetails,
-      );
+      logger.error('Proxy v1 endpoint error : metadataArray is not an array', metaTo.errorDetails);
       // Panic
       throw new PlatformError('Proxy v1 endpoint error : metadataArray is not an array');
     }
@@ -215,7 +212,7 @@
         authErrorCategory: errObj.authErrorCategory,
       }),
     } as DeliveryV1Response;
-    MiscService.logError(errObj.message, metaTo.errorDetails);
+    logger.error(errObj.message, metaTo.errorDetails);
     ErrorReportingService.reportError(error, metaTo.errorContext, resp);
     return resp;
   }
@@ -233,7 +230,7 @@
         authErrorCategory: errObj.authErrorCategory,
       }),
     } as UserDeletionResponse;
-    MiscService.logError(errObj.message, metaTo.errorDetails);
+    logger.error(errObj.message, metaTo.errorDetails);
     ErrorReportingService.reportError(error, metaTo.errorContext, resp);
     return resp;
   }

src/services/misc.ts

@@ -1,12 +1,10 @@
 /* eslint-disable global-require, import/no-dynamic-require */
-import { LoggableExtraData } from '@rudderstack/integrations-lib';
 import fs from 'fs';
 import { Context } from 'koa';
 import path from 'path';
 import { DestHandlerMap } from '../constants/destinationCanonicalNames';
 import { getCPUProfile, getHeapProfile } from '../middleware';
-import { ErrorDetailer, Metadata } from '../types';
-import logger from '../logger';
+import { Metadata } from '../types';
 
 export class MiscService {
   public static getDestHandler(dest: string, version: string) {
@@ -76,9 +74,4 @@ export class MiscService {
   public static async getHeapProfile() {
     return getHeapProfile();
   }
-
-  public static logError(message: string, errorDetailer: ErrorDetailer) {
-    const loggableExtraData: Partial<LoggableExtraData> = logger.getLogMetadata(errorDetailer);
-    logger.error(message || '', loggableExtraData);
-  }
 }

src/util/redis/redisConnector.test.js

@@ -2,7 +2,6 @@ const fs = require('fs');
 const path = require('path');
 const version = 'v0';
 const { RedisDB } = require('./redisConnector');
-const logger = require('../../logger');
 jest.mock('ioredis', () => require('../../../test/__mocks__/redis'));
 
 const sourcesList = ['shopify'];
@@ -56,7 +55,7 @@ describe(`Source Tests`, () => {
     data.forEach((dataPoint, index) => {
       it(`${index}. ${source} - ${dataPoint.description}`, async () => {
         try {
-          const output = await transformer.process(dataPoint.input, logger);
+          const output = await transformer.process(dataPoint.input);
           expect(output).toEqual(dataPoint.output);
         } catch (error) {
           expect(error.message).toEqual(dataPoint.output.error);

src/v0/destinations/google_adwords_enhanced_conversions/networkHandler.js

@@ -39,7 +39,7 @@
       query: queryString,
     };
     const searchStreamEndpoint = `${BASE_ENDPOINT}/${params.customerId}/googleAds:searchStream`;
-    logger.requestLog(`[${destType.toUpperCase()}] conversion enhancement request`, {
+    logger.requestLog(`[${destType.toUpperCase()}] get conversion action id request`, {
       metadata,
       requestDetails: { url: searchStreamEndpoint, body: data, method },
     });
@@ -75,7 +75,7 @@
         `"${JSON.stringify(
           get(gaecConversionActionIdResponse, ERROR_MSG_PATH, '')
             ? get(gaecConversionActionIdResponse, ERROR_MSG_PATH, '')
             : response,
         )} during Google_adwords_enhanced_conversions response transformation"`,
         status,
         {

src/v0/destinations/google_adwords_offline_conversions/utils.js

@@ -89,7 +89,7 @@ const getConversionActionId = async ({ headers, params, metadata }) => {
     });
     searchStreamResponse = processAxiosResponse(searchStreamResponse);
     const { response, status, headers: responseHeaders } = searchStreamResponse;
-    logger.responseLog(`[${destType.toUpperCase()}] get conversion custom variable`, {
+    logger.responseLog(`[${destType.toUpperCase()}] get conversion action id response`, {
       metadata,
       responseDetails: {
         response,