diff --git a/.circleci/config.yml b/.circleci/config.yml index 0fac2db..07b0d3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,8 @@ jobs: steps: - checkout - run: npm install - - run: npm run eslint + - run: npm run lint + - run: npm run test build: docker: - image: circleci/node:10 diff --git a/.gitignore b/.gitignore index c63e8f5..70a8e3a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ node_modules # No Testing Materials data -generateData.js \ No newline at end of file +generateData.js +.nyc* diff --git a/CHANGELOG.md b/CHANGELOG.md index efd404d..19c135f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ This file documents all notable changes in `LogDNA CloudWatch Lambda Function`. The release numbering uses [semantic versioning](http://semver.org). +## v2.2.0 - Released on June 9, 2020 +* Add `LOG_RAW_EVENT` environment variable option to set `line` to raw `event.message` + ## v2.1.0 - Released on November 14, 2019 * Update retry mechanism * Remove message truncation diff --git a/LICENSE b/LICENSE index bd95903..2775016 100755 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 LogDNA, Inc. +Copyright (c) 2020 LogDNA, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index ce8bd37..c6cbf75 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,13 @@ The LogDNA AWS CloudWatch integration relies on [AWS Lambda](https://aws.amazon. * `LOGDNA_HOSTNAME`: Alternative Host Name *(Optional)* * `LOGDNA_TAGS`: Comma-separated Tags *(Optional)* * `LOGDNA_URL`: Custom Ingestion URL *(Optional)* + * `LOG_RAW_EVENT`: Setting `line` to Raw `event.message` *(Optional, Default: false)*: + * It can be enabled by setting `LOG_RAW_EVENT` to `YES` or `TRUE` + * Enabling it moves the following `event`-related `meta` data from the `line` field to the `meta` field: + * `event.type`: `messageType` of `CloudWatch Log` encoded inside `awslogs.data` in `base64` + * `event.id`: `id` of each `CloudWatch Log` encoded inside `awslogs.data` in `base64` + * `log.group`: `LogGroup` where the log is coming from + * `log.stream`: `LogStream` where the log is coming from 4. For Execution role, assign an [IAM user with basic execution permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/getting-started_create-admin-group.html) by choosing an existing role and selecting a role that has permissions to upload logs to Amazon CloudWatch logs. ### Configure your AWS CloudWatch Log Group diff --git a/index.js b/index.js index 2cd5286..1cb6296 100644 --- a/index.js +++ b/index.js @@ -5,26 +5,27 @@ const request = require('request'); const zlib = require('zlib'); // Constants -const MAX_LINE_LENGTH = parseInt(process.env.LOGDNA_MAX_LINE_LENGTH) || 32000; const MAX_REQUEST_TIMEOUT_MS = parseInt(process.env.LOGDNA_MAX_REQUEST_TIMEOUT) || 30000; const FREE_SOCKET_TIMEOUT_MS = parseInt(process.env.LOGDNA_FREE_SOCKET_TIMEOUT) || 300000; const LOGDNA_URL = process.env.LOGDNA_URL || 'https://logs.logdna.com/logs/ingest'; const MAX_REQUEST_RETRIES = parseInt(process.env.LOGDNA_MAX_REQUEST_RETRIES) || 5; const REQUEST_RETRY_INTERVAL_MS = parseInt(process.env.LOGDNA_REQUEST_RETRY_INTERVAL) || 100; +const INTERNAL_SERVER_ERROR = 500; const DEFAULT_HTTP_ERRORS = [ 'ECONNRESET' , 'EHOSTUNREACH' , 'ETIMEDOUT' , 'ESOCKETTIMEDOUT' , 'ECONNREFUSED' - , 'ENOTFOUND']; + , 'ENOTFOUND' +]; -const INTERNAL_SERVER_ERROR = 500; // Get Configuration from Environment Variables const getConfig = () => { const pkg = require('./package.json'); let config = { - UserAgent: `${pkg.name}/${pkg.version}` + log_raw_event: false + , UserAgent: `${pkg.name}/${pkg.version}` }; if (process.env.LOGDNA_KEY) config.key = process.env.LOGDNA_KEY; @@ -33,6 +34,11 @@ const getConfig = () => { config.tags = process.env.LOGDNA_TAGS.split(',').map(tag => tag.trim()).join(','); } + if (process.env.LOG_RAW_EVENT) { + config.log_raw_event = process.env.LOG_RAW_EVENT.toLowerCase(); + config.log_raw_event = config.log_raw_event === 'yes' || config.log_raw_event === 'true'; + } + return config; }; @@ -42,28 +48,35 @@ const parseEvent = (event) => { }; // Prepare the Messages and Options -const prepareLogs = (eventData) => { +const prepareLogs = (eventData, log_raw_event) => { return eventData.logEvents.map((event) => { - return { - line: JSON.stringify({ - message: event.message - , source: 'cloudwatch' - , event: { - type: eventData.messageType - , id: event.id - } - , log: { - group: eventData.logGroup - , stream: eventData.logStream - } - }) - , timestamp: event.timestamp + const eventMetadata = { + event: { + type: eventData.messageType + , id: event.id + }, log: { + group: eventData.logGroup + , stream: eventData.logStream + } + }; + + const eventLog = { + timestamp: event.timestamp , file: eventData.logStream , meta: { owner: eventData.owner , filters: eventData.subscriptionFilters - } + }, line: JSON.stringify(Object.assign({}, { + message: event.message + }, eventMetadata)) }; + + if (log_raw_event) { + eventLog.line = event.message; + eventLog.meta = Object.assign({}, eventLog.meta, eventMetadata); + } + + return eventLog; }); }; @@ -81,17 +94,13 @@ const sendLine = (payload, config, callback) => { , qs: config.tags ? { tags: config.tags , hostname: hostname - } : { - hostname: hostname - } + } : { hostname: hostname } , method: 'POST' , body: JSON.stringify({ e: 'ls' , ls: payload }) - , auth: { - username: config.key - } + , auth: { username: config.key } , headers: { 'Content-Type': 'application/json; charset=UTF-8' , 'user-agent': config.UserAgent @@ -108,8 +117,7 @@ const sendLine = (payload, config, callback) => { times: MAX_REQUEST_RETRIES , interval: (retryCount) => { return REQUEST_RETRY_INTERVAL_MS * Math.pow(2, retryCount); - } - , errorFilter: (errCode) => { + }, errorFilter: (errCode) => { return DEFAULT_HTTP_ERRORS.includes(errCode) || errCode === 'INTERNAL_SERVER_ERROR'; } }, (reqCallback) => { @@ -129,6 +137,15 @@ const sendLine = (payload, config, callback) => { }; // Main Handler -exports.handler = (event, context, callback) => { - return sendLine(prepareLogs(parseEvent(event)), getConfig(), callback); +const handler = (event, context, callback) => { + const config = getConfig(); + return sendLine(prepareLogs(parseEvent(event), config.log_raw_event), config, callback); +}; + +module.exports = { + getConfig + , handler + , parseEvent + , prepareLogs + , sendLine }; diff --git a/package.json b/package.json index 6a784e6..deabf52 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,11 @@ { "name": "logdna-cloudwatch", - "version": "2.1.0", + "version": "2.2.0", "description": "Lambda Functions to Stream Logs from AWS CloudWatch to LogDNA", "main": "index.js", "scripts": { - "eslint": "./node_modules/.bin/eslint -c .eslintrc *.js" + "lint": "./node_modules/.bin/eslint -c .eslintrc index.js", + "test": "tap" }, "dependencies": { "agentkeepalive": "^4.0.2", @@ -12,7 +13,8 @@ "request": "^2.88.0" }, "devDependencies": { - "eslint": "^6.7.2" + "eslint": "^6.7.2", + "tap": "^14.10.7" }, "keywords": [ "lambda", diff --git a/test/index.js b/test/index.js new file mode 100644 index 0000000..1c61462 --- /dev/null +++ b/test/index.js @@ -0,0 +1,146 @@ +// External Modules +const { test } = require('tap'); + +// Internal Modules +const index = require('../index'); +const pkg = require('../package.json'); + +// Constants +const missingKey = 'Missing LogDNA Ingestion Key'; +const hostname = 'sampleCloudWatchHostname'; +const inputTags = ' cloudwatch, logging, test'; +const outputTags = 'cloudwatch,logging,test'; +const sampleKey = '0123456789'; +const rawEvent = { + 'awslogs': { + 'data': 'H4sIAAAAAAAAEzWQQW+DMAyF/wrKmaEkJCbhhjbWCzuBtMNUVSmkNBIQRMKqqep/X6Cb5Ivfs58++45G7ZzqdfMza5Sjt6IpTh9lXReHEsXI3ia9BJnQlHHIhMSEBnmw/WGx6xwcp8Z50M9uN2q/aDUGx2vn/5oYufXs2sXM3tjp3QxeLw7lX6hS47lTz6lTO9i1uynfXkOMe5lsp9Fxzyy/9eS3hTsyXYhOGVCaEsBSgsyEYBkGzrDMAIMQlAq+gQIQSjFhBFgqJOUMAog34WAfoFFOOM8kA0Y5SSH+f0SIb67GRaHq/baosn1UmUlHF7tErxvk5wa56b2Z+iRJ0OP4+AWj9ITzSgEAAA==' + } +}; + +const eventData = { + messageType: 'DATA_MESSAGE' + , owner: '123456789012' + , logGroup: 'sampleGroup' + , logStream: 'testStream' + , subscriptionFilters: [ 'LambdaStream_cloudwatchlogs-node' ] + , logEvents: [{ + id: '34622316099697884706540976068822859012661220141643892546' + , timestamp: 1557946425136 + , message: 'This is Sample Log Line for CloudWatch Logging...' + }] +}; + +const eventMetaData = { + event: { + type: eventData.messageType + , id: eventData.logEvents[0].id + }, log: { + group: eventData.logGroup + , stream: eventData.logStream + } +}; + +// Test parseEvent +test('test parseEvent with the sample test data described in README', (t) => { + t.deepEqual(index.parseEvent(rawEvent), eventData); + t.end(); +}); + +// Test getConfig +test('test getConfig', (t) => { + // Test getConfig without any environment variable set + let config = index.getConfig(); + t.equal(config.key, undefined); + t.equal(config.log_raw_event, false); + t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`); + t.equal(config.hostname, undefined); + t.equal(config.tags, undefined); + + // Set Hostname, Key and Tags + process.env.LOGDNA_HOSTNAME = hostname; + process.env.LOGDNA_TAGS = inputTags; + process.env.LOGDNA_KEY = sampleKey; + config = index.getConfig(); + t.equal(config.key, sampleKey); + t.equal(config.log_raw_event, false); + t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`); + t.equal(config.hostname, hostname); + t.equal(config.tags, outputTags); + + // Set LOG_RAW_EVENT to True + process.env.LOG_RAW_EVENT = 'True'; + config = index.getConfig(); + t.equal(config.key, sampleKey); + t.equal(config.log_raw_event, true); + t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`); + t.equal(config.hostname, hostname); + t.equal(config.tags, outputTags); + + // Unset some environment variables + process.env.LOG_RAW_EVENT = ''; + process.env.LOGDNA_TAGS = ''; + config = index.getConfig(); + t.equal(config.key, sampleKey); + t.equal(config.log_raw_event, false); + t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`); + t.equal(config.hostname, hostname); + t.equal(config.tags, undefined); + + // Set LOG_RAW_EVENT to Yes + process.env.LOG_RAW_EVENT = 'yEs'; + config = index.getConfig(); + t.equal(config.key, sampleKey); + t.equal(config.log_raw_event, true); + t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`); + t.equal(config.hostname, hostname); + t.equal(config.tags, undefined); + + // Finish the test suite + t.end(); +}); + +// Test prepareLogs +test('test prepareLogs', (t) => { + // Without log_raw_event set to true + let eventLog = index.prepareLogs(eventData, false)[0]; + t.assert(eventLog.timestamp < Date.now()); + t.equal(eventLog.file, eventData.logStream); + t.equal(eventLog.meta.owner, eventData.owner); + t.deepEqual(eventLog.meta.filters, eventData.subscriptionFilters); + t.deepEqual(JSON.parse(eventLog.line), Object.assign({ + message: eventData.logEvents[0].message + }, eventMetaData)); + + // With log_raw_event set to true + eventLog = index.prepareLogs(eventData, true)[0]; + t.assert(eventLog.timestamp < Date.now()); + t.equal(eventLog.file, eventData.logStream); + t.equal(eventLog.line, eventData.logEvents[0].message); + t.deepEqual(eventLog.meta, Object.assign({ + owner: eventData.owner + , filters: eventData.subscriptionFilters + }, eventMetaData)); + + // Finish the test suite + t.end(); +}); + +// Test sendLine +test('test sendLine', (t) => { + index.sendLine({ line: eventData.logEvents[0].message }, {}, (error, response) => { + t.equal(error, missingKey); + + // Finish the test suite + t.end(); + }); +}); + +// Test handler +test('test handler', (t) => { + index.sendLine(rawEvent, {}, (error, response) => { + t.equal(error, missingKey); + + // Finish the test suite + t.end(); + }); +});