From 2804065390a6f2cf5a9228ebced5a350590109bf Mon Sep 17 00:00:00 2001 From: rotem-codefresh <88274951+rotem-codefresh@users.noreply.github.com> Date: Tue, 17 Aug 2021 16:32:28 +0300 Subject: [PATCH] fixed bug, added env variables (#712) * fixed bug, added env variables * deleted json files --- .../cli/commands/offline-logs/base.cmd.js | 69 ++++++++----------- .../offline-logs/offload-to-collection.cmd.js | 20 ++++-- .../offline-logs/offload-to-file.cmd.js | 27 +++++++- .../cli/commands/offline-logs/utils.js | 41 ++++++++--- package.json | 2 +- 5 files changed, 97 insertions(+), 62 deletions(-) diff --git a/lib/interface/cli/commands/offline-logs/base.cmd.js b/lib/interface/cli/commands/offline-logs/base.cmd.js index 8230cf63b..744940730 100644 --- a/lib/interface/cli/commands/offline-logs/base.cmd.js +++ b/lib/interface/cli/commands/offline-logs/base.cmd.js @@ -1,46 +1,33 @@ -const yargs = require('yargs'); -const Command = require('../../Command'); +const yargs = require("yargs"); +const Command = require("../../Command"); const command = new Command({ - command: 'offline-logs', - root: true, - description: 'Manages offline logs', - webDocs: { - category: 'Logs category', - subCategory: 'Logs sub category', - title: 'Archives old logs to file or collection.', - }, - builder: (yargs) => { - // set options which are used in both sub-commands - return yargs - .option('uri', { - describe: "Mongodb URI", - demandOption: true, - type: "string", - }) - .option('db', { - describe: "Database name", - demandOption: true, - type: "string", - }) - .option('collections', { - alias: "c", - describe: "Source collections names", - default: ["logs", "metadata"], - array: true, - type: "string", - }) - .option('cutoffDate', { - alias: "cod", - describe: - "The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).", - demandOption: true, - type: "string", - }); - }, - handler: () => { - yargs.showHelp(); - }, + command: "offline-logs", + root: true, + description: "Manages offline logs", + webDocs: { + category: "Logs category", + subCategory: "Logs sub category", + title: "Archives old logs to file or collection.", + }, + builder: (yargs) => { + // set options which are used in both sub-commands + return yargs + .env("RUNTIME_MONGO") + .option("uri", { + describe: + "Mongodb URI. If not provided, will be parsed from environment variables.", + type: "string", + }) + .option("db", { + describe: + "Database name. If not provided, will be parsed from environment variables.", + type: "string", + }); + }, + handler: () => { + yargs.showHelp(); + }, }); module.exports = command; diff --git a/lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js b/lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js index b8566f0fb..39a4995f7 100644 --- a/lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js +++ b/lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js @@ -2,7 +2,7 @@ const { MongoClient, ObjectId } = require("mongodb"); const moment = require('moment') const Command = require('../../Command'); const cmd = require('./base.cmd'); -const { objectIdFromDate } = require('./utils') +const { objectIdFromDate, defaultCollections } = require('./utils') const offloadToCollection = async function(sourceDBObj, collection, targetDB, cutoffDate) { const sourceCollectionObj = sourceDBObj.collection(collection) @@ -10,13 +10,13 @@ const offloadToCollection = async function(sourceDBObj, collection, targetDB, cu const cutoffDateObj = moment(cutoffDate) .add(1, 'days') - .startOf("day").toDate() + .startOf("day") if(!cutoffDateObj.isValid()){ throw new Error('please enter a valid date in ISO 8601 format') } - const cutoffDateId = objectIdFromDate(cutoffDateObj) + const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate()) var result = sourceCollectionObj.aggregate([ { $match: { _id: { $lte: ObjectId(cutoffDateId) } } }, @@ -31,6 +31,7 @@ const offloadToCollection = async function(sourceDBObj, collection, targetDB, cu await result.toArray() if (!result.cursorState.killed){ + console.info(`logs from '${collection}' were archived to '${targetCollection}'`) await sourceCollectionObj.deleteMany({_id: {$lte: ObjectId(cutoffDateId)}}) } else { @@ -53,21 +54,28 @@ const command = new Command({ describe: "Target database name, if none inserted, db will be defined as target.", type: "string", }) - .example('codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --c logs foo --cod "2021-07-08" '), + .option('cutoffDate', { + alias: "cod", + describe: + "The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).", + demandOption: true, + type: "string", + }) + .example('codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --c logs foo --cod "2021-07-08" '), handler: async (argv) => { const { uri, db, - collections, targetDB, cutoffDate, } = argv + const client = new MongoClient(uri); try{ await client.connect() const failedCollections = []; const sourceDBObj = client.db(db); - const promises = collections.map( async (collection) => { + const promises = defaultCollections.map( async (collection) => { try{ await offloadToCollection(sourceDBObj, collection, targetDB || db, cutoffDate); } catch (error) { diff --git a/lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js b/lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js index 5b3e15670..584b928a9 100644 --- a/lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js +++ b/lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js @@ -3,8 +3,7 @@ const { MongoClient, ObjectId } = require("mongodb"); const moment = require('moment') const Command = require('../../Command'); const cmd = require('./base.cmd'); -const {objectIdFromDate, writeLogsToFile, checkRemnant} = require('./utils') -const { boolean } = require("yargs"); +const {objectIdFromDate, writeLogsToFile, checkRemnant, defaultCollections} = require('./utils'); const offloadToFile = async function(database, collection, chunkDuration, cutoffDate, path) { const collectionObj = database.collection(collection); @@ -93,12 +92,34 @@ const command = new Command({ default: 1, type: "number", }) + .option('cutoffDate', { + alias: "cod", + describe: + "The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).", + demandOption: true, + type: "string", + }) .option('path', { describe: "Directory path to which archive files will be saved.", default: ".", type: "string", }) - .example('codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections logs foo --cod "2021-07-08" --chdur 3 --path "./" '), + .option('collections', { + alias: 'c', + describe: "Source collections names", + default: defaultCollections, + array: true, + type: "string", + coerce(arg) { + for (const collection of arg) { + if (!collection.includes('logs') && !collection.includes('metadata')) { + throw new Error ('invalid collection name') + } + return arg + } + } + }) + .example('codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections archive-logs --cod "2021-07-08" --chdur 3 --path "./" '), handler: async (argv) => { const { uri, diff --git a/lib/interface/cli/commands/offline-logs/utils.js b/lib/interface/cli/commands/offline-logs/utils.js index 8412e648d..46e764eb1 100644 --- a/lib/interface/cli/commands/offline-logs/utils.js +++ b/lib/interface/cli/commands/offline-logs/utils.js @@ -1,24 +1,43 @@ -const moment = require('moment'); +const readline = require("readline"); +const moment = require("moment"); -const { join } = require('path'); -const fs = require('fs'); +const { join } = require("path"); +const fs = require("fs"); + +const defaultCollections = ["logs", "metadata"]; // converts date to objectId for filter purposes const objectIdFromDate = function (date) { - return Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000"; + return Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000"; }; -const writeLogsToFile = function(upperBound, lowerBound, collection, logsToArchive, path) { - const date = moment(upperBound).subtract(1, "days") - const fileDateRange = `${moment(lowerBound).format('YYYY-MM-DD')}-${date.format('YYYY-MM-DD')}` +const writeLogsToFile = function ( + upperBound, + lowerBound, + collection, + logsToArchive, + path +) { + const date = moment(upperBound).subtract(1, "days"); + const fileDateRange = `${moment(lowerBound).format( + "YYYY-MM-DD" + )}-${date.format("YYYY-MM-DD")}`; const fileName = `${fileDateRange}-${collection}.json`; const absPath = join(path, fileName); const data = JSON.stringify(logsToArchive); fs.writeFileSync(absPath, data); + console.info(`logs from collection '${collection}' were offloaded to '${absPath}'`) +}; + +const checkRemnant = function (lowerBound, cutoffDateObj) { + return Math.ceil(cutoffDateObj.diff(lowerBound, "days")); }; -const checkRemnant = function(lowerBound, cutoffDateObj) { - return Math.ceil(cutoffDateObj.diff(lowerBound , "days" )); -} -module.exports = {objectIdFromDate, writeLogsToFile, checkRemnant}; + +module.exports = { + objectIdFromDate, + writeLogsToFile, + checkRemnant, + defaultCollections, +}; diff --git a/package.json b/package.json index 90ed2f5c2..2991727d3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codefresh", - "version": "0.76.0", + "version": "0.77.0", "description": "Codefresh command line utility", "main": "index.js", "preferGlobal": true,