From 48d149549ffc4ea4beee61c8f003edf0bd95e9e2 Mon Sep 17 00:00:00 2001 From: Vishal Date: Fri, 16 Feb 2024 17:09:54 +0530 Subject: [PATCH] Cleanup (#9) * remove Bun worker * replace command classes with module --- README.md | 8 +- cmd/Taskfile.yml | 6 +- cmd/filterer/index.ts | 231 --------------------- cmd/filterer/reader.worker.ts | 84 -------- cmd/src/commands/filterer/index.ts | 192 ++++++++++++++++++ cmd/src/commands/filterer/worker.ts | 53 +++++ cmd/src/commands/summary/index.ts | 243 +++++++++++++++++++++++ cmd/{ => src/commands}/summary/worker.ts | 7 +- cmd/{ => src}/main.ts | 39 ++-- cmd/{ => src}/utils/cmd-runner/index.ts | 0 cmd/{ => src}/utils/file-helper/index.ts | 0 cmd/{ => src}/utils/worker-pool/index.ts | 0 cmd/summary/index.ts | 236 ---------------------- cmd/tsconfig.json | 14 +- ui/src/pages/normalize/index.tsx | 4 +- ui/tsconfig.json | 1 + 16 files changed, 529 insertions(+), 589 deletions(-) delete mode 100644 cmd/filterer/index.ts delete mode 100644 cmd/filterer/reader.worker.ts create mode 100644 cmd/src/commands/filterer/index.ts create mode 100644 cmd/src/commands/filterer/worker.ts create mode 100644 cmd/src/commands/summary/index.ts rename cmd/{ => src/commands}/summary/worker.ts (91%) rename cmd/{ => src}/main.ts (69%) rename cmd/{ => src}/utils/cmd-runner/index.ts (100%) rename cmd/{ => src}/utils/file-helper/index.ts (100%) rename cmd/{ => src}/utils/worker-pool/index.ts (100%) delete mode 100644 cmd/summary/index.ts diff --git a/README.md b/README.md index 3c0bf8c..663b0e1 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,9 @@ For details, refer to its [README.md](https://github.com/vish9812/analog/blob/ma ## Prerequisite: -Install [Bun](https://bun.sh/docs/installation) to run as CLI. +- [Bun](https://bun.sh/docs/installation) is needed to run the CLI. +- [Python 3](https://www.python.org/downloads/) is needed to run the UI. + - Python will be automatically installed if you run the Analog UI app with the `analog` script. ## Getting Started @@ -73,13 +75,13 @@ Following are the 3 must have keys in the logs: - timestamp - msg -Example Format for JSON logs: +Expected Format for JSON logs: ``` {"timestamp":"2023-10-16 10:13:16.710 +11:00","level":"debug","msg":"Received HTTP request","dynamicKey1":"value 1","dynamicKey2":"value 2"} ``` -Example Format for plain-text logs: +Expected Format for plain-text logs: ``` debug [2023-10-16 10:13:16.710 +11:00] Received HTTP request dynamicKey1="value 1" dynamicKey2=value 2 diff --git a/cmd/Taskfile.yml b/cmd/Taskfile.yml index 93d6aeb..d701c58 100644 --- a/cmd/Taskfile.yml +++ b/cmd/Taskfile.yml @@ -11,8 +11,8 @@ tasks: cmds: - > bun build - ./main.ts - ./summary/worker.ts - ./filterer/reader.worker.ts + ./src/main.ts + ./src/commands/summary/worker.ts + ./src/commands/filterer/worker.ts --outdir ./cli --target bun diff --git a/cmd/filterer/index.ts b/cmd/filterer/index.ts deleted file mode 100644 index 624d247..0000000 --- a/cmd/filterer/index.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { readdir } from "node:fs/promises"; -import * as path from "node:path"; -import { parseArgs } from "util"; -import { cpus } from "node:os"; -import type { JSONLog } from "@al/models/logData"; -import type { ICmd } from "@al/cmd/utils/cmd-runner"; -import readerWorker, { - type IReaderRequest, - type IReaderResponse, -} from "./reader.worker"; - -class Filterer implements ICmd { - private minTime: string | undefined; - private maxTime: string | undefined; - private inFolderPath: string = ""; - private outFileName: string = ""; - private filteredLogs: JSONLog[] = []; - private filePaths: string[] = []; - private workerPromises: Promise[] = []; - - // Warn: Workers are not production ready yet. - // Experimental Note at the top of the page: https://bun.sh/docs/api/workers - private readonly workerURL = new URL("reader.worker.ts", import.meta.url) - .href; - - help(): void { - console.log(` - Filters all files from a given folder within a time range and generates a single time-sorted log file. - The timestamps format must match the format available in the log files. - - Usage: - - analog --filter(-f) [arguments] - - The arguments are: - - --minTime(-x) Filters out logs with timestamps earlier than the specified minimum time(inclusive). - - --maxTime(-y) Filters out logs with timestamps equal or later than the specified maximum time(exclusive). - - --inFolderPath(-i) Specifies the path to the folder containing the log files. - The folder should only contain log files or nested folders with log files. - - --outFileName(-o) Specifies the name of the filtered log file to generate. - If the file already exists, its content will be overridden. - - Example: - - analog -f -x "2024-01-25 19:00:00.000 +00:00" -y "2024-01-25 19:05:00.000 +00:00" -i "/path/to/logs/folder" -o "/path/to/filtered/log/filename.log" - `); - } - - async run(): Promise { - this.parseFlags(); - await this.processLogs(); - } - - private parseFlags() { - const { values: flags } = parseArgs({ - args: Bun.argv, - options: { - filter: { - type: "boolean", - short: "f", - }, - minTime: { - type: "string", - short: "x", - }, - maxTime: { - type: "string", - short: "y", - }, - inFolderPath: { - type: "string", - short: "i", - }, - outFileName: { - type: "string", - short: "o", - }, - }, - strict: true, - allowPositionals: true, - }); - - if (!flags.inFolderPath) throw new Error("Pass input logs folder path."); - if (!flags.outFileName) throw new Error("Pass output logs file name."); - if (!flags.minTime && !flags.maxTime) { - throw new Error( - "Pass at least one flag for filtering by time: minTime or maxTime." - ); - } - - this.inFolderPath = flags.inFolderPath; - this.outFileName = flags.outFileName; - this.minTime = flags.minTime; - this.maxTime = flags.maxTime; - } - - private async processLogs() { - this.filePaths = await this.getFilesRecursively(); - - await this.readFiles(); - - if (this.workerPromises.length) { - await Promise.all(this.workerPromises); - } - - this.sortLogs(); - await this.writeContent(); - } - - private async readFiles() { - let hasWorker = true; - try { - new Worker(this.workerURL); - } catch (err) { - // Issue: https://github.com/oven-sh/bun/issues/7901 - hasWorker = false; - } - - console.log("Processing files parallely : ", hasWorker); - hasWorker ? this.readFilesParallely() : await this.readFilesSerially(); - } - - private processFileResponse(fileLogs?: JSONLog[]) { - if (!fileLogs?.length) return; - - let idx = this.filteredLogs.length; - - // Instead of pushing one-by-one, increase length by fixed amount in one-go. - this.filteredLogs.length += fileLogs.length; - fileLogs.forEach((l) => (this.filteredLogs[idx++] = l)); - } - - // TODO: Need it for the compiled cli only as the cli currently doesn't support Workers - // Issue: https://github.com/oven-sh/bun/issues/7901 - private async readFilesSerially() { - for (const filePath of this.filePaths) { - const fileLogs = await readerWorker.processFile({ - filePath, - minTime: this.minTime, - maxTime: this.maxTime, - }); - this.processFileResponse(fileLogs); - } - } - - private readFilesParallely() { - const maxWorkers = Math.min( - Math.max(cpus().length - 1, 1), - this.filePaths.length - ); - - for (let i = 0; i < maxWorkers; i++) { - const promise = new Promise((res) => { - const worker = new Worker(this.workerURL); - - worker.addEventListener("close", () => { - res(); - }); - - worker.onmessage = (event: Bun.MessageEvent) => { - if (event.data.work === readerWorker.Work.Ask) { - this.processFileResponse(event.data.filteredLogs); - - const filePath = this.filePaths.pop(); - const message: IReaderRequest = filePath - ? { - work: readerWorker.Work.Begin, - workData: { - filePath, - minTime: this.minTime, - maxTime: this.maxTime, - }, - } - : { work: readerWorker.Work.End }; - - worker.postMessage(message); - } - }; - }); - - this.workerPromises.push(promise); - } - } - - private async getFilesRecursively(): Promise { - const fileList: string[] = []; - - async function readDirectory(currentPath: string) { - const files = await readdir(currentPath, { withFileTypes: true }); - - for (const f of files) { - const filePath = path.join(currentPath, f.name); - - if (f.isDirectory()) { - // If it's a directory, recursively read its contents - await readDirectory(filePath); - } else { - // If it's a file, add its path to the list - fileList.push(filePath); - } - } - } - - await readDirectory(this.inFolderPath); - - return fileList; - } - - private sortLogs() { - console.log("Starting Sorting..."); - this.filteredLogs.sort((a, b) => - a.timestamp < b.timestamp ? -1 : a.timestamp > b.timestamp ? 1 : 0 - ); - console.log("Finished Sorting."); - } - - private async writeContent() { - console.log("Joining Logs"); - const allContent = this.filteredLogs - .map((l) => JSON.stringify(l)) - .join("\r\n"); - console.log("Writing Logs"); - await Bun.write(this.outFileName, allContent); - } -} - -export default Filterer; diff --git a/cmd/filterer/reader.worker.ts b/cmd/filterer/reader.worker.ts deleted file mode 100644 index 4488ba6..0000000 --- a/cmd/filterer/reader.worker.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { JSONLog } from "@al/models/logData"; -import normalizer from "@al/services/normalizer"; - -// prevents TS errors -declare var self: Worker; - -enum Work { - Ask, - Begin, - End, -} - -interface IAnalogWorker { - work: Work; -} - -interface IReaderRequest extends IAnalogWorker { - workData?: IWorkData; -} - -interface IReaderResponse extends IAnalogWorker { - filteredLogs?: JSONLog[]; -} - -interface IWorkData { - filePath: string; - minTime: string | undefined; - maxTime: string | undefined; -} - -self.onmessage = async (event: MessageEvent) => { - if (event.data.work === Work.Begin) { - const data = event.data.workData!; - - const filteredLogs = await processFile(data); - - const res: IReaderResponse = { - work: Work.Ask, - filteredLogs, - }; - postMessage(res); - } else if (event.data.work === Work.End) { - process.exit(); - } -}; - -const res: IReaderResponse = { work: Work.Ask }; -postMessage(res); - -async function processFile(data: IWorkData): Promise { - const logsArr: JSONLog[] = []; - let linesCount = 0; - - const text = await Bun.file(data.filePath).text(); - - const filterer = ({ timestamp }: JSONLog) => - !!( - (data.minTime && timestamp < data.minTime) || - (data.maxTime && timestamp >= data.maxTime) - ); - - const logsGeneratorFn = normalizer.parse( - text, - normalizer.getParserOptions(text), - filterer - ); - for (const jsonLog of logsGeneratorFn()) { - if (!jsonLog) continue; - logsArr.push(jsonLog); - linesCount++; - } - - console.info(`Filter Lines Count: ${linesCount} for File: ${data.filePath} `); - - return logsArr; -} - -const readerWorker = { - Work, - processFile, -}; - -export default readerWorker; -export type { IReaderRequest, IReaderResponse }; diff --git a/cmd/src/commands/filterer/index.ts b/cmd/src/commands/filterer/index.ts new file mode 100644 index 0000000..e6c435b --- /dev/null +++ b/cmd/src/commands/filterer/index.ts @@ -0,0 +1,192 @@ +import { parseArgs } from "util"; +import { cpus } from "node:os"; +import type { ITask, IResult } from "./worker"; +import type { ICmd } from "@al/cmd/utils/cmd-runner"; +import type { JSONLog } from "@al/ui/models/logData"; +import fileHelper from "@al/cmd/utils/file-helper"; +import WorkerPool from "@al/cmd/utils/worker-pool"; + +let workerURL = new URL("worker.ts", import.meta.url); + +const flags = { + minTime: "0", + maxTime: "z", + inFolderPath: "", + outFileName: "", +}; + +const filteredLogs: JSONLog[] = []; + +function help(): void { + console.log(` +Filters all files from a given folder within a time range and generates a single time-sorted log file. +The timestamps format must match the format available in the log files. + +Caution: Passing a big time range could lead to keeping millions of log lines in RAM which may lead to slowness. + Also generating a single big file of more than 300k lines may not be that useful or easy to analyze. + So, start with small time ranges that you're interested in and then increase the range accordingly. + +Usage: + + bun run ./cli/main.js --filter [arguments] + +The arguments are: + + -x, --minTime + Filters out logs with timestamps earlier than the specified minimum time(inclusive). + Optional: if maxTime has been provided. + + -y, --maxTime + Filters out logs with timestamps equal or later than the specified maximum time(exclusive). + Optional: if minTime has been provided. + + -i, --inFolderPath + Specifies the path to the folder containing the log files. + The folder should only contain log files or nested folders with log files. + + -o, --outFileName + Specifies the name of the filtered log file to generate. + If the file already exists, its content will be overridden. + +Example: + + bun run ./cli/main.js -f -x "2024-01-25 19:00:00.000 +00:00" -y "2024-01-25 19:05:00.000 +00:00" -i "/path/to/logs/folder" -o "/path/to/filtered/log/filename.log" + `); +} + +async function run(): Promise { + const workerFile = Bun.file(workerURL); + if (!(await workerFile.exists())) { + // Path for the bundled code + workerURL = new URL("commands/filterer/worker.js", import.meta.url); + } + + parseFlags(); + + await processLogs(); +} + +function parseFlags() { + const { values } = parseArgs({ + args: Bun.argv, + options: { + filter: { + type: "boolean", + short: "f", + }, + minTime: { + type: "string", + short: "x", + default: "", + }, + maxTime: { + type: "string", + short: "y", + default: "", + }, + inFolderPath: { + type: "string", + short: "i", + }, + outFileName: { + type: "string", + short: "o", + }, + }, + strict: true, + allowPositionals: true, + }); + + if (!values.inFolderPath) throw new Error("Pass input logs folder path."); + if (!values.outFileName) throw new Error("Pass output logs file name."); + if (!values.minTime && !values.maxTime) { + throw new Error( + "Pass at least one flag for filtering by time: minTime or maxTime." + ); + } + + flags.inFolderPath = values.inFolderPath; + flags.outFileName = values.outFileName; + if (values.minTime) flags.minTime = values.minTime; + if (values.maxTime) flags.maxTime = values.maxTime; +} + +async function processLogs() { + const filePaths = await fileHelper.getFilesRecursively(flags.inFolderPath); + + console.log("=========Begin Read Files========="); + await readFiles(filePaths); + console.log("=========End Read Files========="); + + sortLogs(); + + await writeContent(); +} + +function readFiles(filePaths: string[]) { + return new Promise((res, rej) => { + const maxWorkers = Math.min( + Math.max(cpus().length - 1, 1), + filePaths.length + ); + + const pool = new WorkerPool(workerURL, maxWorkers); + + let finishedTasks = 0; + for (const filePath of filePaths) { + pool.runTask( + { + filePath, + minTime: flags.minTime, + maxTime: flags.maxTime, + }, + async (err, result) => { + if (err) { + console.error("Failed for file: ", filePath); + rej(); + } + + processFileResponse(result); + + if (++finishedTasks === filePaths.length) { + await pool.close(); + res(); + } + } + ); + } + }); +} + +function processFileResponse(result: IResult) { + const fileLogs = result.filteredLogs; + if (!fileLogs.length) return; + + let idx = filteredLogs.length; + + // Instead of pushing one-by-one, increase length by fixed amount in one-go. + filteredLogs.length += fileLogs.length; + fileLogs.forEach((fl) => (filteredLogs[idx++] = fl)); +} + +function sortLogs() { + console.log("Sorting Logs"); + filteredLogs.sort((a, b) => + a.timestamp < b.timestamp ? -1 : a.timestamp > b.timestamp ? 1 : 0 + ); +} + +async function writeContent() { + console.log("Total Logs matched: ", filteredLogs.length); + console.log("Joining Logs"); + const allContent = filteredLogs.map((l) => JSON.stringify(l)).join("\r\n"); + console.log("Writing Logs"); + await Bun.write(flags.outFileName, allContent); +} + +const filterer: ICmd = { + help, + run, +}; + +export default filterer; diff --git a/cmd/src/commands/filterer/worker.ts b/cmd/src/commands/filterer/worker.ts new file mode 100644 index 0000000..f4a67e6 --- /dev/null +++ b/cmd/src/commands/filterer/worker.ts @@ -0,0 +1,53 @@ +import type { JSONLog } from "@al/ui/models/logData"; +import normalizer from "@al/ui/services/normalizer"; +import { parentPort } from "node:worker_threads"; + +interface ITask { + filePath: string; + minTime: string | undefined; + maxTime: string | undefined; +} + +interface IResult { + filteredLogs: JSONLog[]; +} + +// Thread Code +if (parentPort) { + parentPort.on("message", async (task: ITask) => { + const result = await processFile(task); + parentPort!.postMessage(result); + }); +} + +async function processFile(data: ITask): Promise { + const result: IResult = { + filteredLogs: [], + }; + + const text = await Bun.file(data.filePath).text(); + + const filterer = ({ timestamp }: JSONLog) => + !!( + (data.minTime && timestamp < data.minTime) || + (data.maxTime && timestamp >= data.maxTime) + ); + + const logsGeneratorFn = normalizer.parse( + text, + normalizer.getParserOptions(text), + filterer + ); + for (const jsonLog of logsGeneratorFn()) { + if (!jsonLog) continue; + result.filteredLogs.push(jsonLog); + } + + console.log( + `${result.filteredLogs.length} lines matched in File: ${data.filePath} ` + ); + + return result; +} + +export type { ITask, IResult }; diff --git a/cmd/src/commands/summary/index.ts b/cmd/src/commands/summary/index.ts new file mode 100644 index 0000000..98cecb6 --- /dev/null +++ b/cmd/src/commands/summary/index.ts @@ -0,0 +1,243 @@ +import prettyBytes from "pretty-bytes"; +import { Table } from "console-table-printer"; +import { parseArgs } from "util"; +import { cpus } from "node:os"; +import type { ICmd } from "@al/cmd/utils/cmd-runner"; +import { type ITask, type IResult } from "./worker"; +import WorkerPool from "@al/cmd/utils/worker-pool"; +import fileHelper from "@al/cmd/utils/file-helper"; +import type { + GroupedMsg, + Summary as SummaryData, + SummaryMap, +} from "@al/ui/models/logData"; +import LogData from "@al/ui/models/logData"; + +let workerURL = new URL("worker.ts", import.meta.url); + +interface IStats extends Omit { + minTimeFile: string; + maxTimeFile: string; +} + +const stats: IStats = { + maxTime: "0", + maxTimeFile: "", + minTime: "z", + minTimeFile: "", + size: 0, + dataMap: { + httpCodes: new Map(), + jobs: new Map(), + msgs: new Map(), + plugins: new Map(), + }, +}; + +const flags = { + inFolderPath: "", + topLogsCount: 30, +}; + +function help(): void { + console.log(` +Summary provides a summary view of all the log files. + +Usage: + + bun run ./cli/main.js --summary [arguments] + +The arguments are: + + -i, --inFolderPath + Specifies the path to the folder containing the log files. + The folder should only contain log files or nested folders with log files. + + -t, --top + Specifies the maximum number of top logs you see. + Default: 30 + +Example: + + bun run ./cli/main.js -s -i "/path/to/logs/folder" + `); +} + +async function run(): Promise { + const workerFile = Bun.file(workerURL); + if (!(await workerFile.exists())) { + // Path for the bundled code + workerURL = new URL("commands/summary/worker.js", import.meta.url); + } + + parseFlags(); + + await processLogs(); +} + +function parseFlags() { + const { values } = parseArgs({ + args: Bun.argv, + options: { + summary: { + type: "boolean", + short: "s", + }, + inFolderPath: { + type: "string", + short: "i", + }, + top: { + type: "string", + short: "t", + }, + }, + strict: true, + allowPositionals: true, + }); + + if (!values.inFolderPath) throw new Error("Pass input logs folder path."); + + flags.inFolderPath = values.inFolderPath; + if (values.top) flags.topLogsCount = +values.top; +} + +async function processLogs() { + const filePaths = await fileHelper.getFilesRecursively(flags.inFolderPath); + + console.log("=========Begin Read Files========="); + await readFiles(filePaths); + console.log("=========End Read Files========="); + + const summary = LogData.initSummary(stats.dataMap); + + writeContent(summary); +} + +function readFiles(filePaths: string[]) { + return new Promise((res, rej) => { + const maxWorkers = Math.min( + Math.max(cpus().length - 1, 1), + filePaths.length + ); + + const pool = new WorkerPool(workerURL, maxWorkers); + + let finishedTasks = 0; + for (const filePath of filePaths) { + pool.runTask({ filePath }, async (err, result) => { + if (err) { + console.error("Failed for file: " + filePath); + return rej(err); + } + + processFileResponse(result); + + if (++finishedTasks === filePaths.length) { + await pool.close(); + return res(); + } + }); + } + }); +} + +function processFileResponse(fileStats: IResult) { + if (fileStats.maxTime > stats.maxTime) { + stats.maxTime = fileStats.maxTime; + stats.maxTimeFile = fileStats.filePath; + } + + if (fileStats.minTime < stats.minTime) { + stats.minTime = fileStats.minTime; + stats.minTimeFile = fileStats.filePath; + } + + stats.size += fileStats.size; + + initSummaryMap(fileStats.dataMap); +} + +function initSummaryMap(dataMap: SummaryMap) { + mergeIntoOverallMap(dataMap.httpCodes, stats.dataMap.httpCodes); + mergeIntoOverallMap(dataMap.jobs, stats.dataMap.jobs); + mergeIntoOverallMap(dataMap.msgs, stats.dataMap.msgs); + mergeIntoOverallMap(dataMap.plugins, stats.dataMap.plugins); +} + +function mergeIntoOverallMap( + fileMap: Map, + overallMap: Map +) { + for (const [k, v] of fileMap) { + if (!overallMap.has(k)) { + overallMap.set(k, { + msg: v.msg, + hasErrors: false, + logs: [], + logsCount: 0, + }); + } + + const grpOverall = overallMap.get(k)!; + grpOverall.hasErrors = grpOverall.hasErrors || v.hasErrors; + grpOverall.logsCount += v.logsCount!; + } +} + +function writeContent(summary: SummaryData) { + console.log(); + + stats.size = prettyBytes(stats.size) as any; + + let table = new Table({ + title: "Overall summary of all the logs", + columns: [ + { name: "minTime" }, + { name: "minTimeFile" }, + { name: "maxTime" }, + { name: "maxTimeFile" }, + { name: "totalUniqueLogs" }, + { name: "size" }, + ], + disabledColumns: ["dataMap"], + }); + table.addRow( + { + ...stats, + totalUniqueLogs: summary.msgs.length, + }, + { color: "green" } + ); + table.printTable(); + + writeGroupedMsgs(summary.msgs, "Top Logs"); + writeGroupedMsgs(summary.httpCodes, "HTTP Codes"); + writeGroupedMsgs(summary.jobs, "Jobs"); + writeGroupedMsgs(summary.plugins, "Plugins"); +} + +function writeGroupedMsgs(grpMsgs: GroupedMsg[], title: string) { + console.log(); + + const table = new Table({ + columns: [ + { name: "msg", title: title, alignment: "left" }, + { name: "logsCount", title: "Count" }, + ], + disabledColumns: ["hasErrors", "logs"], + }); + + for (const grp of grpMsgs.slice(0, flags.topLogsCount)) { + table.addRow(grp, { color: grp.hasErrors ? "red" : "green" }); + } + + table.printTable(); +} + +const summary: ICmd = { + help, + run, +}; + +export default summary; diff --git a/cmd/summary/worker.ts b/cmd/src/commands/summary/worker.ts similarity index 91% rename from cmd/summary/worker.ts rename to cmd/src/commands/summary/worker.ts index 4dfd838..3d90611 100644 --- a/cmd/summary/worker.ts +++ b/cmd/src/commands/summary/worker.ts @@ -1,6 +1,9 @@ import { parentPort } from "node:worker_threads"; -import LogData, { type GroupedMsg, type SummaryMap } from "@al/models/logData"; -import normalizer from "@al/services/normalizer"; +import LogData, { + type GroupedMsg, + type SummaryMap, +} from "@al/ui/models/logData"; +import normalizer from "@al/ui/services/normalizer"; interface ITask { filePath: string; diff --git a/cmd/main.ts b/cmd/src/main.ts similarity index 69% rename from cmd/main.ts rename to cmd/src/main.ts index 9cf9376..0867c55 100644 --- a/cmd/main.ts +++ b/cmd/src/main.ts @@ -1,9 +1,9 @@ import { parseArgs } from "util"; -import type { ICmd } from "./utils/cmd-runner"; -import Filterer from "./filterer"; -import Summary from "./summary"; // @ts-ignore import figlet from "figlet"; +import filterer from "./commands/filterer"; +import summary from "./commands/summary"; +import type { ICmd } from "./utils/cmd-runner"; let cmd: ICmd; let isHelp = false; @@ -33,9 +33,9 @@ if (typeof flags.help === "boolean" && flags.help) { } if (typeof flags.filter === "boolean" && flags.filter) { - cmd = new Filterer(); + cmd = filterer; } else if (typeof flags.summary === "boolean" && flags.summary) { - cmd = new Summary(); + cmd = summary; } else { help(); process.exit(0); @@ -49,10 +49,6 @@ if (isHelp) { console.log("========Finished========"); } -// TODO: Bug: Something is keeping the main process alive, so exiting forcefully. -console.log("Beyonder..."); -process.exit(0); - function help() { console.log(figlet.textSync(`ANALOG`)); console.log(` @@ -71,24 +67,27 @@ function help() { `); console.log(` - Run analog as cli for analyzing multiple log files. - - Usage: - - bun run ./cli/main.js [arguments] +Run analog as cli for analyzing multiple log files. - The commands are: - +Usage: + + bun run ./cli/main.js [arguments] + +The commands are: + -s, --summary provides a summary view of all the log files. -f, --filter filters all files from a given folder within a time range and generate a single time-sorted log file. - Use "bun run ./cli/main.js --help " for more information about a command. - Example: bun run ./cli/main.js --help --filter +Use "bun run ./cli/main.js --help " for more information about a command. + +Example: + + bun run ./cli/main.js --help --filter - Caution: Processing multiple files will need at least twice the space as the logs files size. - For example, if you are analyzing 4GB of logs make sure you have 8GB of *free* RAM left for smoother processing. +Caution: Processing multiple files will need at least twice the space as the logs files size. + For example, if you are analyzing 4GB of logs make sure you have 8GB of *free* RAM left for smoother processing. `); } diff --git a/cmd/utils/cmd-runner/index.ts b/cmd/src/utils/cmd-runner/index.ts similarity index 100% rename from cmd/utils/cmd-runner/index.ts rename to cmd/src/utils/cmd-runner/index.ts diff --git a/cmd/utils/file-helper/index.ts b/cmd/src/utils/file-helper/index.ts similarity index 100% rename from cmd/utils/file-helper/index.ts rename to cmd/src/utils/file-helper/index.ts diff --git a/cmd/utils/worker-pool/index.ts b/cmd/src/utils/worker-pool/index.ts similarity index 100% rename from cmd/utils/worker-pool/index.ts rename to cmd/src/utils/worker-pool/index.ts diff --git a/cmd/summary/index.ts b/cmd/summary/index.ts deleted file mode 100644 index 6bb719c..0000000 --- a/cmd/summary/index.ts +++ /dev/null @@ -1,236 +0,0 @@ -import prettyBytes from "pretty-bytes"; -import { Table } from "console-table-printer"; -import { parseArgs } from "util"; -import { cpus } from "node:os"; -import type { ICmd } from "@al/cmd/utils/cmd-runner"; -import { type ITask, type IResult } from "./worker"; -import WorkerPool from "@al/cmd/utils/worker-pool"; -import fileHelper from "@al/cmd/utils/file-helper"; -import type { - GroupedMsg, - Summary as SummaryData, - SummaryMap, -} from "@al/models/logData"; -import LogData from "@al/models/logData"; - -let workerURL = new URL("worker.ts", import.meta.url); - -interface IStats extends Omit { - minTimeFile: string; - maxTimeFile: string; -} - -class Summary implements ICmd { - private stats: IStats = { - maxTime: "0", - maxTimeFile: "", - minTime: "z", - minTimeFile: "", - size: 0, - dataMap: { - httpCodes: new Map(), - jobs: new Map(), - msgs: new Map(), - plugins: new Map(), - }, - }; - - private flags = { - inFolderPath: "", - }; - - private filePaths: string[] = []; - - help(): void { - console.log(` - Summary provides a summary view of all the log files. - - Usage: - - bun run ./cli/main.js --summary [arguments] - - The arguments are: - - --inFolderPath(-i) Specifies the path to the folder containing the log files. - The folder should only contain log files or nested folders with log files. - - Example: - - bun run ./cli/main.js -s -i "/path/to/logs/folder" - `); - } - - async run(): Promise { - const workerFile = Bun.file(workerURL); - if (!(await workerFile.exists())) { - // Path for the bundled code - workerURL = new URL("summary/worker.js", import.meta.url); - } - - this.parseFlags(); - - await this.processLogs(); - } - - private parseFlags() { - const { values } = parseArgs({ - args: Bun.argv, - options: { - summary: { - type: "boolean", - short: "s", - }, - inFolderPath: { - type: "string", - short: "i", - }, - }, - strict: true, - allowPositionals: true, - }); - - if (!values.inFolderPath) throw new Error("Pass input logs folder path."); - - this.flags.inFolderPath = values.inFolderPath; - } - - private async processLogs() { - this.filePaths = await fileHelper.getFilesRecursively( - this.flags.inFolderPath - ); - - console.log("=========Begin Read Files========="); - await this.readFiles(); - console.log("=========End Read Files========="); - - const summary = LogData.initSummary(this.stats.dataMap); - - this.writeContent(summary); - } - - private readFiles() { - return new Promise((res, rej) => { - const maxWorkers = Math.min( - Math.max(cpus().length - 1, 1), - this.filePaths.length - ); - - const pool = new WorkerPool(workerURL, maxWorkers); - - let finishedTasks = 0; - for (const filePath of this.filePaths) { - pool.runTask({ filePath }, async (err, result) => { - if (err) { - console.error("Failed for file: ", filePath); - rej(); - } - - await this.processFileResponse(result); - - if (++finishedTasks === this.filePaths.length) { - await pool.close(); - res(); - } - }); - } - }); - } - - private async processFileResponse(fileStats: IResult) { - if (fileStats.maxTime > this.stats.maxTime) { - this.stats.maxTime = fileStats.maxTime; - this.stats.maxTimeFile = fileStats.filePath; - } - - if (fileStats.minTime < this.stats.minTime) { - this.stats.minTime = fileStats.minTime; - this.stats.minTimeFile = fileStats.filePath; - } - - this.stats.size += fileStats.size; - - this.initSummaryMap(fileStats.dataMap); - } - - private initSummaryMap(dataMap: SummaryMap) { - Summary.mergeIntoOverallMap( - dataMap.httpCodes, - this.stats.dataMap.httpCodes - ); - Summary.mergeIntoOverallMap(dataMap.jobs, this.stats.dataMap.jobs); - Summary.mergeIntoOverallMap(dataMap.msgs, this.stats.dataMap.msgs); - Summary.mergeIntoOverallMap(dataMap.plugins, this.stats.dataMap.plugins); - } - - private static mergeIntoOverallMap( - fileMap: Map, - overallMap: Map - ) { - for (const [k, v] of fileMap) { - if (!overallMap.has(k)) { - overallMap.set(k, { - msg: v.msg, - hasErrors: false, - logs: [], - logsCount: 0, - }); - } - - const grpOverall = overallMap.get(k)!; - grpOverall.hasErrors = grpOverall.hasErrors || v.hasErrors; - grpOverall.logsCount += v.logsCount!; - } - } - - private writeContent(summary: SummaryData) { - console.log(); - - this.stats.size = prettyBytes(this.stats.size) as any; - - let table = new Table({ - title: "Overall summary of all the logs", - columns: [ - { name: "minTime" }, - { name: "minTimeFile" }, - { name: "maxTime" }, - { name: "maxTimeFile" }, - { name: "totalUniqueLogs" }, - { name: "size" }, - ], - disabledColumns: ["dataMap"], - }); - table.addRow( - { - ...this.stats, - totalUniqueLogs: summary.msgs.length, - }, - { color: "green" } - ); - table.printTable(); - - Summary.writeGroupedMsgs(summary.msgs, "Top Logs"); - Summary.writeGroupedMsgs(summary.httpCodes, "HTTP Codes"); - Summary.writeGroupedMsgs(summary.jobs, "Jobs"); - Summary.writeGroupedMsgs(summary.plugins, "Plugins"); - } - - private static writeGroupedMsgs(grpMsgs: GroupedMsg[], title: string) { - console.log(); - - const table = new Table({ - columns: [ - { name: "msg", title: title, alignment: "left" }, - { name: "logsCount", title: "Count" }, - ], - disabledColumns: ["hasErrors", "logs"], - }); - - for (const grp of grpMsgs.slice(0, 30)) { - table.addRow(grp, { color: grp.hasErrors ? "red" : "green" }); - } - - table.printTable(); - } -} - -export default Summary; diff --git a/cmd/tsconfig.json b/cmd/tsconfig.json index 328abc0..8bec695 100644 --- a/cmd/tsconfig.json +++ b/cmd/tsconfig.json @@ -18,14 +18,12 @@ "strict": true, "noFallthroughCasesInSwitch": true, "forceConsistentCasingInFileNames": true, - "baseUrl": "..", "paths": { - "@al/cmd/*": ["cmd/*"], - "@al/components/*": ["ui/src/components/*"], - "@al/models/*": ["ui/src/models/*"], - "@al/pages/*": ["ui/src/pages/*"], - "@al/services/*": ["ui/src/services/*"], - "@al/utils/*": ["ui/src/utils/*"] + "@al/cmd/*": ["./src/*"], + "@al/ui/models/*": ["../ui/src/models/*"], + "@al/ui/services/*": ["../ui/src/services/*"], + "@al/ui/utils/*": ["../ui/src/utils/*"] } - } + }, + "include": ["src/**/*", "../ui/src/**/*"] } diff --git a/ui/src/pages/normalize/index.tsx b/ui/src/pages/normalize/index.tsx index 04b27ea..6f47802 100644 --- a/ui/src/pages/normalize/index.tsx +++ b/ui/src/pages/normalize/index.tsx @@ -57,12 +57,12 @@ function Normalize() { - timestamp - msg `} -

Example Format for JSON logs:

+

Expected Format for JSON logs:

{`
 {"timestamp":"2023-10-16 10:13:16.710 +11:00","level":"debug","msg":"Received HTTP request","dynamicKey1":"value 1","dynamicKey2":"value 2"}
         `}
-

Example Format for plain-text logs:

+

Expected Format for plain-text logs:

{`
 debug [2023-10-16 10:13:16.710 +11:00] Received HTTP request dynamicKey1="value 1" dynamicKey2=value 2
         `}
diff --git a/ui/tsconfig.json b/ui/tsconfig.json index c04de56..ad062e2 100644 --- a/ui/tsconfig.json +++ b/ui/tsconfig.json @@ -13,6 +13,7 @@ "isolatedModules": true, "baseUrl": "src", "paths": { + /* If required, add paths in the cmd tsconfig.json as well */ "@al/components/*": ["components/*"], "@al/models/*": ["models/*"], "@al/pages/*": ["pages/*"],