From 4853ea25c47bf08db55fb3a9a6f56eb142aef27d Mon Sep 17 00:00:00 2001 From: Vishal Date: Fri, 17 Nov 2023 16:05:19 +0530 Subject: [PATCH] Add more Insights (#3) * add more insights * update Readme.md --- README.md | 9 +- package.json | 2 +- src/components/filters/index.tsx | 142 +++++++---- src/components/filters/useViewModel.test.tsx | 241 ++++++++++++++----- src/components/filters/useViewModel.tsx | 94 ++++++-- src/models/logData.test.ts | 103 +++++--- src/models/logData.ts | 139 +++++++++-- src/services/comparer.test.ts | 35 +-- src/services/comparer.ts | 14 +- src/services/normalizer.ts | 126 +++++----- 10 files changed, 630 insertions(+), 275 deletions(-) diff --git a/README.md b/README.md index 8a35c4e..aaa40bb 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,20 @@ Analog is a powerful tool designed for analyzing and visualizing log files. It p ## Features -- **Top Logs**: Quickly identify and analyze the most frequently occurring log entries. +- **Summary View**: Quickly gain insights into your log file with the Summary View. It provides frequencies of the following key aspects: + + - Top Logs + - HTTP Codes + - Jobs + - Plugins - **Filter Logs**: - **Filter by Timestamp**: Specify a start and end timestamp to narrow down your log analysis. - **Regex Search**: Perform regular expression searches to find specific log entries. - **Search Combinations**: Perform normal searches with advanced combination of `Contains/Not Contains` and `AND/OR` operators. - - **Top Logs**: Select entries in the Top Logs to view all the related logs together. - **Errors Only**: Isolate and focus on error log entries. + - **Summary View**: Filter on any of the key aspect of the Summary View. - **Logs Context**: Even when a filter is applied, you can access the context around the current log entry. diff --git a/package.json b/package.json index e7a8744..6ec551b 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "test": "vitest --silent", "test-logs": "vitest", "coverage": "vitest run --coverage --silent", - "build": "vite build && cp analog.sh analog && cp analog.bat analog && zip -r analog.zip analog", + "build": "vite build && cp analog.sh analog && cp analog.ps1 analog && zip -r analog.zip analog", "preview": "vite preview" }, "license": "MIT", diff --git a/src/components/filters/index.tsx b/src/components/filters/index.tsx index 781c5a3..5801b6a 100644 --- a/src/components/filters/index.tsx +++ b/src/components/filters/index.tsx @@ -7,7 +7,11 @@ import { Switch, TextField, } from "@suid/material"; -import useViewModel, { SearchTerm, FiltersProps } from "./useViewModel"; +import useViewModel, { + SearchTerm, + FiltersProps, + GridsRefs, +} from "./useViewModel"; import { AgGridSolidRef } from "ag-grid-solid"; import { GridOptions } from "ag-grid-community"; import { Accessor, For, Show } from "solid-js"; @@ -26,14 +30,31 @@ const texts = { notContains: "Not Contains", }; +interface GridsOptions { + msgs: GridOptions; + httpCodes: GridOptions; + jobs: GridOptions; + plugins: GridOptions; + added: GridOptions; + removed: GridOptions; +} + function Filters(props: FiltersProps) { - let topLogsGridRef = {} as AgGridSolidRef; - let addedLogsGridRef = {} as AgGridSolidRef; - let removedLogsGridRef = {} as AgGridSolidRef; + const gridsRefs: GridsRefs = { + msgs: {} as AgGridSolidRef, + httpCodes: {} as AgGridSolidRef, + jobs: {} as AgGridSolidRef, + plugins: {} as AgGridSolidRef, + added: {} as AgGridSolidRef, + removed: {} as AgGridSolidRef, + }; - let { + const { filters, - topLogs, + msgs, + httpCodes, + jobs, + plugins, addedLogs, removedLogs, setFilters, @@ -63,30 +84,34 @@ function Filters(props: FiltersProps) { ], rowSelection: "multiple", suppressRowClickSelection: true, - onSelectionChanged: handleLogsSelectionChanged, + onSelectionChanged: () => handleLogsSelectionChanged(gridsRefs), getRowStyle: (params) => params.data?.hasErrors ? { background: "#FFBFBF" } : undefined, }; - const topLogsGridOptions: GridOptions = { - ...commonGridOptions, - rowData: topLogs(), - }; - - const addedLogsGridOptions: GridOptions = { - ...commonGridOptions, - rowData: addedLogs(), - }; - - const removedLogsGridOptions: GridOptions = { - ...commonGridOptions, - rowData: removedLogs(), - columnDefs: [ - { ...commonGridOptions.columnDefs![0], checkboxSelection: undefined }, - { ...commonGridOptions.columnDefs![1] }, - ], - rowSelection: undefined, - onSelectionChanged: undefined, + const gridsOptions: GridsOptions = { + msgs: { ...commonGridOptions, rowData: msgs() }, + jobs: { ...commonGridOptions, rowData: jobs() }, + plugins: { ...commonGridOptions, rowData: plugins() }, + added: { ...commonGridOptions, rowData: addedLogs() }, + httpCodes: { + ...commonGridOptions, + rowData: httpCodes(), + columnDefs: [ + { ...commonGridOptions.columnDefs![0], flex: 1 }, + { ...commonGridOptions.columnDefs![1], flex: 1 }, + ], + }, + removed: { + ...commonGridOptions, + rowData: removedLogs(), + columnDefs: [ + { ...commonGridOptions.columnDefs![0], checkboxSelection: undefined }, + { ...commonGridOptions.columnDefs![1] }, + ], + rowSelection: undefined, + onSelectionChanged: undefined, + }, }; function handleEnterKey(e: KeyboardEvent) { @@ -113,7 +138,7 @@ function Filters(props: FiltersProps) { } /> setFilters("terms", i(), "value", val)} onKeyDown={handleEnterKey} @@ -164,7 +189,7 @@ function Filters(props: FiltersProps) { @@ -185,30 +210,51 @@ function Filters(props: FiltersProps) { + + + + + + + + + - - - - - - - - - - + + + + + + + + + + ); } diff --git a/src/components/filters/useViewModel.test.tsx b/src/components/filters/useViewModel.test.tsx index fb42480..eff3ea4 100644 --- a/src/components/filters/useViewModel.test.tsx +++ b/src/components/filters/useViewModel.test.tsx @@ -1,29 +1,67 @@ import { createRoot } from "solid-js"; -import useViewModel from "./useViewModel"; -import { FiltersData, FiltersProps } from "./useViewModel"; +import useViewModel, { GridsRefs, defaultFilters } from "./useViewModel"; +import { FiltersProps } from "./useViewModel"; import comparer from "@al/services/comparer"; -import LogData, { GroupedMsg } from "@al/models/logData"; +import LogData, { Summary } from "@al/models/logData"; describe("useViewModel", () => { - const topLogs: GroupedMsg[] = [ - { - logs: [{}, {}, {}], - hasErrors: false, - msg: "grp1", - }, - { - logs: [{}], - hasErrors: true, - msg: "grp2", - }, - { - logs: [{}, {}], - hasErrors: true, - msg: "grp3", - }, - ]; + const summary: Summary = { + msgs: [ + { + logs: [{}, {}, {}], + hasErrors: false, + msg: "grp1", + }, + { + logs: [{}], + hasErrors: true, + msg: "grp2", + }, + { + logs: [{}, {}], + hasErrors: true, + msg: "grp3", + }, + ], + httpCodes: [ + { + logs: [{}, {}, {}], + hasErrors: false, + msg: "404", + }, + { + logs: [{}], + hasErrors: true, + msg: "200", + }, + ], + jobs: [ + { + logs: [{}, {}, {}], + hasErrors: false, + msg: "job-1", + }, + { + logs: [{}], + hasErrors: true, + msg: "job-2", + }, + ], + plugins: [ + { + logs: [{}, {}, {}], + hasErrors: false, + msg: "plugin-1", + }, + { + logs: [{}], + hasErrors: true, + msg: "plugin-2", + }, + ], + }; - comparer.removed = [topLogs[0], topLogs[2]]; + comparer.removed = [summary.msgs[0], summary.msgs[2]]; comparer.added = [ { logs: [{}, {}, {}, {}, {}], @@ -37,21 +75,6 @@ describe("useViewModel", () => { }, ]; - const defaultFilters: FiltersData = { - startTime: "", - endTime: "", - regex: "", - terms: [ - { - and: true, - contains: true, - value: "", - }, - ], - logs: [], - errorsOnly: false, - }; - let props: FiltersProps; beforeEach(() => { @@ -60,7 +83,7 @@ describe("useViewModel", () => { }; const lastLogData = new LogData(); - lastLogData.topLogs = topLogs; + lastLogData.summary = summary; vi.spyOn(comparer, "last").mockReturnValue(lastLogData); }); @@ -74,8 +97,13 @@ describe("useViewModel", () => { expect(vm.addedLogs(), "addedLogs").toEqual(comparer.added); expect(vm.removedLogs(), "removedLogs").toEqual(comparer.removed); - expect(vm.topLogs(), "topLogs").toEqual(comparer.last().topLogs); - expect(vm.filters, "filters").toEqual(defaultFilters); + expect(vm.msgs(), "msgs").toEqual(comparer.last().summary.msgs); + expect(vm.httpCodes(), "httpCodes").toEqual( + comparer.last().summary.httpCodes + ); + expect(vm.jobs(), "jobs").toEqual(comparer.last().summary.jobs); + expect(vm.plugins(), "plugins").toEqual(comparer.last().summary.plugins); + expect(vm.filters, "filters").toEqual(defaultFilters()); dispose(); }); @@ -102,8 +130,14 @@ describe("useViewModel", () => { setFilterModel: vi.fn(), }, }); - const topLogsGridRef = getGrid(); - const addedLogsGridRef = getGrid(); + const gridsRefs: GridsRefs = { + msgs: getGrid() as any, + httpCodes: getGrid() as any, + jobs: getGrid() as any, + plugins: getGrid() as any, + added: getGrid() as any, + removed: getGrid() as any, + }; const vm = useViewModel(props); vm.setFilters(() => ({ @@ -115,39 +149,113 @@ describe("useViewModel", () => { })); expect(vm.filters.regex, "regex").toEqual("some regex"); - vm.handleResetClick(topLogsGridRef as any, addedLogsGridRef as any); + vm.handleResetClick(gridsRefs); - expect(vm.filters, "filters").toEqual(defaultFilters); + expect(vm.filters, "filters").toEqual(defaultFilters()); expect(vm.addedLogs(), "addedLogs").toEqual(comparer.added); expect(vm.removedLogs(), "removedLogs").toEqual(comparer.removed); - expect(vm.topLogs(), "topLogs").toEqual(comparer.last().topLogs); + expect(vm.msgs(), "msgs").toEqual(comparer.last().summary.msgs); + expect(vm.httpCodes(), "httpCodes").toEqual( + comparer.last().summary.httpCodes + ); + expect(vm.jobs(), "jobs").toEqual(comparer.last().summary.jobs); + expect(vm.plugins(), "plugins").toEqual(comparer.last().summary.plugins); + expect(props.onFiltersChange, "onFiltersChange").toBeCalledWith( - defaultFilters + defaultFilters() ); - expect(topLogsGridRef.api.deselectAll).toHaveBeenCalledOnce(); - expect(topLogsGridRef.api.setFilterModel).toHaveBeenCalledOnce(); - expect(addedLogsGridRef.api.deselectAll).toHaveBeenCalledOnce(); - expect(addedLogsGridRef.api.setFilterModel).toHaveBeenCalledOnce(); + expect( + gridsRefs.msgs.api.deselectAll, + "msgs.api.deselectAll" + ).toHaveBeenCalledOnce(); + expect( + gridsRefs.msgs.api.setFilterModel, + "msgs.api.setFilterModel" + ).toHaveBeenCalledOnce(); + expect( + gridsRefs.httpCodes.api.deselectAll, + "httpCodes.api.deselectAll" + ).toHaveBeenCalledOnce(); + expect( + gridsRefs.httpCodes.api.setFilterModel, + "httpCodes.api.setFilterModel" + ).toHaveBeenCalledOnce(); + expect( + gridsRefs.jobs.api.deselectAll, + "jobs.api.deselectAll" + ).toHaveBeenCalledOnce(); + expect( + gridsRefs.jobs.api.setFilterModel, + "jobs.api.setFilterModel" + ).toHaveBeenCalledOnce(); + expect( + gridsRefs.plugins.api.deselectAll, + "plugins.api.deselectAll" + ).toHaveBeenCalledOnce(); + expect( + gridsRefs.plugins.api.setFilterModel, + "plugins.api.setFilterModel" + ).toHaveBeenCalledOnce(); + dispose(); }); }); test("handleLogsSelectionChanged", () => { createRoot((dispose) => { - const rows = [{ logs: [{}, {}] }, { logs: [{}] }]; - const logs = rows.flatMap((r) => r.logs); - const selectionEvent = { - api: { - getSelectedRows: () => rows, - }, + const gridsRefs: GridsRefs = { + msgs: { + api: { + getSelectedRows: () => [ + { logs: [{ id: 1 }, { id: 2 }] }, + { logs: [{ id: 3 }] }, + ], + }, + } as any, + httpCodes: { + api: { + getSelectedRows: () => [ + { logs: [{ id: 1 }, { id: 2 }] }, + { logs: [{ id: 4 }] }, + ], + }, + } as any, + jobs: { + api: { + getSelectedRows: () => [{ logs: [{ id: 5 }] }], + }, + } as any, + plugins: { + api: { + getSelectedRows: () => [ + { logs: [{ id: 4 }, { id: 5 }] }, + { logs: [{ id: 6 }] }, + ], + }, + } as any, + added: { + api: { + getSelectedRows: () => [{ logs: [{ id: 1 }, { id: 7 }] }], + }, + } as any, + removed: undefined as any, }; const vm = useViewModel(props); - vm.handleLogsSelectionChanged(selectionEvent as any); + vm.handleLogsSelectionChanged(gridsRefs); + const logs = [ + { id: 1 }, + { id: 2 }, + { id: 3 }, + { id: 4 }, + { id: 5 }, + { id: 6 }, + { id: 7 }, + ]; expect(vm.filters.logs, "filters.logs").toEqual(logs); expect(props.onFiltersChange, "onFiltersChange").toBeCalledWith({ - ...defaultFilters, + ...defaultFilters(), logs, }); @@ -162,16 +270,22 @@ describe("useViewModel", () => { const vm = useViewModel(props); vm.handleErrorsOnlyChange(checked); - const errTopLogs = [topLogs[1], topLogs[2]]; + const errMsgs = [summary.msgs[1], summary.msgs[2]]; + const errHTTPCodes = [summary.httpCodes[1]]; + const errJobs = [summary.jobs[1]]; + const errPlugins = [summary.plugins[1]]; const errAddedTopLogs = [comparer.added[1]]; const errRemovedTopLogs = [comparer.removed[1]]; expect(vm.filters.errorsOnly, "errorsOnly").toEqual(checked); expect(vm.addedLogs(), "addedLogs").toEqual(errAddedTopLogs); expect(vm.removedLogs(), "removedLogs").toEqual(errRemovedTopLogs); - expect(vm.topLogs(), "topLogs").toEqual(errTopLogs); + expect(vm.msgs(), "msgs").toEqual(errMsgs); + expect(vm.httpCodes(), "httpCodes").toEqual(errHTTPCodes); + expect(vm.jobs(), "jobs").toEqual(errJobs); + expect(vm.plugins(), "plugins").toEqual(errPlugins); expect(props.onFiltersChange, "onFiltersChange").toBeCalledWith({ - ...defaultFilters, + ...defaultFilters(), errorsOnly: checked, }); @@ -188,9 +302,12 @@ describe("useViewModel", () => { expect(vm.filters.errorsOnly, "errorsOnly").toEqual(checked); expect(vm.addedLogs(), "addedLogs").toEqual(comparer.added); expect(vm.removedLogs(), "removedLogs").toEqual(comparer.removed); - expect(vm.topLogs(), "topLogs").toEqual(comparer.last().topLogs); + expect(vm.msgs(), "msgs").toEqual(summary.msgs); + expect(vm.httpCodes(), "httpCodes").toEqual(summary.httpCodes); + expect(vm.jobs(), "jobs").toEqual(summary.jobs); + expect(vm.plugins(), "plugins").toEqual(summary.plugins); expect(props.onFiltersChange, "onFiltersChange").toBeCalledWith( - defaultFilters + defaultFilters() ); dispose(); diff --git a/src/components/filters/useViewModel.tsx b/src/components/filters/useViewModel.tsx index 1fdb080..bd21f3a 100644 --- a/src/components/filters/useViewModel.tsx +++ b/src/components/filters/useViewModel.tsx @@ -1,14 +1,22 @@ import { createSignal } from "solid-js"; import { createStore } from "solid-js/store"; -import { SelectionChangedEvent } from "ag-grid-community"; import { AgGridSolidRef } from "ag-grid-solid"; import comparer from "@al/services/comparer"; -import { JSONLogs, GroupedMsg } from "@al/models/logData"; +import LogData, { JSONLogs, GroupedMsg, JSONLog } from "@al/models/logData"; interface FiltersProps { onFiltersChange: (filters: FiltersData) => void; } +interface GridsRefs { + msgs: AgGridSolidRef; + httpCodes: AgGridSolidRef; + jobs: AgGridSolidRef; + plugins: AgGridSolidRef; + added: AgGridSolidRef; + removed: AgGridSolidRef; +} + interface SearchTerm { and: boolean; contains: boolean; @@ -41,11 +49,14 @@ function defaultFilters(): FiltersData { }; } -const errorFilterFn = (prev: GroupedMsg[]) => prev.filter((m) => m.hasErrors); - function useViewModel(props: FiltersProps) { const [filters, setFilters] = createStore(defaultFilters()); - const [topLogs, setTopLogs] = createSignal(comparer.last().topLogs); + const [msgs, setMsgs] = createSignal(comparer.last().summary.msgs); + const [httpCodes, setHTTPCodes] = createSignal( + comparer.last().summary.httpCodes + ); + const [jobs, setJobs] = createSignal(comparer.last().summary.jobs); + const [plugins, setPlugins] = createSignal(comparer.last().summary.plugins); const [addedLogs, setAddedLogs] = createSignal(comparer.added); const [removedLogs, setRemovedLogs] = createSignal(comparer.removed); @@ -53,37 +64,64 @@ function useViewModel(props: FiltersProps) { props.onFiltersChange(filters); } - function handleResetClick( - topLogsGridRef: AgGridSolidRef, - addedLogsGridRef: AgGridSolidRef - ) { + function handleResetClick(gridsRefs: GridsRefs) { setFilters(defaultFilters()); handleErrorsOnlyChange(false); - topLogsGridRef.api.setFilterModel(null); - topLogsGridRef.api.deselectAll(); + + gridsRefs.msgs.api.setFilterModel(null); + gridsRefs.msgs.api.deselectAll(); + + gridsRefs.httpCodes.api.setFilterModel(null); + gridsRefs.httpCodes.api.deselectAll(); + + gridsRefs.jobs.api.setFilterModel(null); + gridsRefs.jobs.api.deselectAll(); + + gridsRefs.plugins.api.setFilterModel(null); + gridsRefs.plugins.api.deselectAll(); + if (addedLogs().length > 0) { - addedLogsGridRef.api.setFilterModel(null); - addedLogsGridRef.api.deselectAll(); + gridsRefs.added.api.setFilterModel(null); + gridsRefs.added.api.deselectAll(); } } - function handleLogsSelectionChanged(e: SelectionChangedEvent) { - setFilters( - "logs", - e.api.getSelectedRows().flatMap((n) => n.logs) - ); + function handleLogsSelectionChanged(gridsRefs: GridsRefs) { + const map = new Map(); + populateMap(gridsRefs.msgs); + populateMap(gridsRefs.httpCodes); + populateMap(gridsRefs.jobs); + populateMap(gridsRefs.plugins); + + if (addedLogs().length > 0) { + populateMap(gridsRefs.added); + } + + setFilters("logs", [...map.values()]); handleFiltersChange(); + + function populateMap(gridRef: AgGridSolidRef) { + for (const r of gridRef.api.getSelectedRows() as GroupedMsg[]) { + r.logs.forEach((l) => map.set(l[LogData.logKeys.id], l)); + } + } } function handleErrorsOnlyChange(checked: boolean) { setFilters("errorsOnly", checked); if (checked) { - setTopLogs(errorFilterFn); - setAddedLogs(errorFilterFn); - setRemovedLogs(errorFilterFn); + setMsgs(LogData.errorFilterFn); + setHTTPCodes(LogData.errorFilterFn); + setJobs(LogData.errorFilterFn); + setPlugins(LogData.errorFilterFn); + setAddedLogs(LogData.errorFilterFn); + setRemovedLogs(LogData.errorFilterFn); } else { - setTopLogs(comparer.last().topLogs); + setMsgs(comparer.last().summary.msgs); + setHTTPCodes(comparer.last().summary.httpCodes); + setJobs(comparer.last().summary.jobs); + setPlugins(comparer.last().summary.plugins); setAddedLogs(comparer.added); setRemovedLogs(comparer.removed); } @@ -106,9 +144,12 @@ function useViewModel(props: FiltersProps) { return { filters, - topLogs, - addedLogs: addedLogs, - removedLogs: removedLogs, + msgs, + httpCodes, + jobs, + plugins, + addedLogs, + removedLogs, setFilters, handleFiltersChange, handleLogsSelectionChanged, @@ -119,4 +160,5 @@ function useViewModel(props: FiltersProps) { } export default useViewModel; -export type { SearchTerm, FiltersData, FiltersProps }; +export { defaultFilters }; +export type { SearchTerm, FiltersData, FiltersProps, GridsRefs }; diff --git a/src/models/logData.test.ts b/src/models/logData.test.ts index d369858..893e941 100644 --- a/src/models/logData.test.ts +++ b/src/models/logData.test.ts @@ -1,5 +1,4 @@ -import stringsUtils from "@al/utils/strings"; -import LogData, { GroupedMsg, JSONLog, LogsGenerator } from "./logData"; +import LogData, { JSONLog, LogsGenerator, Summary } from "./logData"; describe("isErrorLog", () => { it.each([ @@ -17,6 +16,13 @@ describe("isErrorLog", () => { }, expected: true, }, + { + test: "Error key", + log: { + [LogData.logKeys.Error]: "Error key", + }, + expected: true, + }, { test: "both error level and key", log: { @@ -39,23 +45,30 @@ describe("isErrorLog", () => { describe("init", () => { const cutOffLen = LogData["msgCutOffLen"]; + const getCutOffMsg = LogData["getCutOffMsg"]; it("init", () => { const log0 = { [LogData.logKeys.error]: "some error", [LogData.logKeys.msg]: "has errors", + [LogData.logKeys.status_code]: "404", + [LogData.logKeys.plugin_id]: "plugin-1", parentKey1: "k1", } as JSONLog; const log0String = getJSONString(log0); const log1 = { [LogData.logKeys.level]: "error", [LogData.logKeys.msg]: "dbg msg", + [LogData.logKeys.status]: "200", + [LogData.logKeys.plugin_id]: "plugin-2", + [LogData.logKeys.worker]: "job-1", parentKey1: "k1", } as JSONLog; const log1String = getJSONString(log1); const log2 = { [LogData.logKeys.level]: "info", [LogData.logKeys.msg]: "abc ".repeat(cutOffLen) + "group 1", + [LogData.logKeys.scheduler_name]: "job-1", parentKey1: { childKey1: 11, childKey2: 12, @@ -65,6 +78,8 @@ describe("init", () => { const log3 = { [LogData.logKeys.level]: "error", [LogData.logKeys.msg]: "abc ".repeat(cutOffLen) + "group 1", + [LogData.logKeys.status_code]: "404", + [LogData.logKeys.worker]: "job-2", parentKey1: "k1", } as JSONLog; const log3String = getJSONString(log3); @@ -115,59 +130,74 @@ describe("init", () => { ]; expect(logData.logs, "logs").toEqual(expectedLogs); - function getCutOffMsg(log: any) { - return stringsUtils - .cleanText(log[LogData.logKeys.msg]) - .substring(0, cutOffLen) - .trim(); - } - const expectedTopLogsMap = new Map([ - [ - getCutOffMsg(log0), + const expectedSummary: Summary = { + msgs: [ { msg: getCutOffMsg(log0), logs: [expectedLogs[0]] as any, hasErrors: true, }, - ], - [ - getCutOffMsg(log1), { msg: getCutOffMsg(log1), logs: [expectedLogs[1]] as any, hasErrors: true, }, - ], - [ - getCutOffMsg(log2), { msg: getCutOffMsg(log2), logs: [expectedLogs[2], expectedLogs[3]] as any, hasErrors: true, }, - ], - [ - getCutOffMsg(log4), { msg: getCutOffMsg(log4), logs: [expectedLogs[4], expectedLogs[5]] as any, hasErrors: false, }, - ], - ]); - expect(logData.topLogsMap.size, "topLogsMap.size").toEqual( - expectedTopLogsMap.size - ); - - for (const [k, v] of expectedTopLogsMap) { - const topLog = logData.topLogsMap.get(k); - expect(topLog, "topLog").toBeTruthy(); - expect(topLog?.hasErrors, "topLogsMap.hasErrors").toEqual(v.hasErrors); - expect(topLog?.logs, "topLogsMap.logs").toEqual(v.logs); - } + ].sort(LogData.sortByLogsFn), + httpCodes: [ + { + msg: log0[LogData.logKeys.status_code], + logs: [expectedLogs[0], expectedLogs[3]] as any, + hasErrors: true, + }, + { + msg: log1[LogData.logKeys.status], + logs: [expectedLogs[1]] as any, + hasErrors: true, + }, + ].sort(LogData.sortByMsgFn), + jobs: [ + { + msg: log1[LogData.logKeys.worker], + logs: [expectedLogs[1], expectedLogs[2]] as any, + hasErrors: true, + }, + { + msg: log3[LogData.logKeys.worker], + logs: [expectedLogs[3]] as any, + hasErrors: true, + }, + ].sort(LogData.sortByLogsFn), + plugins: [ + { + msg: log0[LogData.logKeys.plugin_id], + logs: [expectedLogs[0]] as any, + hasErrors: true, + }, + { + msg: log1[LogData.logKeys.plugin_id], + logs: [expectedLogs[1]] as any, + hasErrors: true, + }, + ].sort(LogData.sortByLogsFn), + }; - expect(logData.topLogs, "topLogs").toEqual( - [...expectedTopLogsMap.values()].sort(LogData.sortComparerFn) + expect(logData.summary.msgs, "summary.msgs").toEqual(expectedSummary.msgs); + expect(logData.summary.httpCodes, "summary.httpCodes").toEqual( + expectedSummary.httpCodes + ); + expect(logData.summary.jobs, "summary.jobs").toEqual(expectedSummary.jobs); + expect(logData.summary.plugins, "summary.plugins").toEqual( + expectedSummary.plugins ); const expectedKeys = [ @@ -176,6 +206,11 @@ describe("init", () => { LogData.logKeys.msg, LogData.logKeys.error, LogData.logKeys.level, + LogData.logKeys.plugin_id, + LogData.logKeys.scheduler_name, + LogData.logKeys.worker, + LogData.logKeys.status_code, + LogData.logKeys.status, "parentKey1.childKey2", "parentKey1.childKey1", "parentKey1", diff --git a/src/models/logData.ts b/src/models/logData.ts index c3cd020..4cf72bd 100644 --- a/src/models/logData.ts +++ b/src/models/logData.ts @@ -10,6 +10,20 @@ interface GroupedMsg { hasErrors: boolean; } +interface Summary { + msgs: GroupedMsg[]; + httpCodes: GroupedMsg[]; + jobs: GroupedMsg[]; + plugins: GroupedMsg[]; +} + +interface SummaryMap { + msgs: Map; + httpCodes: Map; + jobs: Map; + plugins: Map; +} + type LogsGenerator = Generator; class LogData { @@ -19,12 +33,21 @@ class LogData { }; logs: JSONLogs = []; - topLogs: GroupedMsg[] = []; - topLogsMap = new Map(); keys: string[] = []; - static readonly sortComparerFn = (a: GroupedMsg, b: GroupedMsg) => + summary: Summary = { + httpCodes: [], + jobs: [], + msgs: [], + plugins: [], + }; + + static readonly sortByLogsFn = (a: GroupedMsg, b: GroupedMsg) => b.logs.length - a.logs.length; + static readonly sortByMsgFn = (a: GroupedMsg, b: GroupedMsg) => + a.msg >= b.msg ? 1 : -1; + static readonly errorFilterFn = (msgs: GroupedMsg[]) => + msgs.filter((m) => m.hasErrors); private static readonly msgCutOffLen = 80; static readonly logKeys = { @@ -34,37 +57,52 @@ class LogData { msg: "msg", level: "level", error: "error", + Error: "Error", + plugin_id: "plugin_id", + worker: "worker", + scheduler_name: "scheduler_name", + status_code: "status_code", + status: "status", }; private static readonly levels = { error: "error", }; - init(file: File, iteratorFunc: () => LogsGenerator) { + init(file: File, logsGeneratorFn: () => LogsGenerator) { this.initFileInfo(file); + const summaryMap: SummaryMap = { + httpCodes: new Map(), + jobs: new Map(), + msgs: new Map(), + plugins: new Map(), + }; + const keysSet = new Set(); let count = 0; - for (const log of iteratorFunc()) { + for (const log of logsGeneratorFn()) { if (log == null) { console.warn("non-supported log format."); continue; } - this.addLog(log); + log[LogData.logKeys.fullData] = JSON.stringify(log); + this.logs.push(log); log[LogData.logKeys.id] = count++ as any; - this.initTopLogsMap(log); + LogData.initSummaryMap(log, summaryMap); LogData.initKeysSet(log, keysSet); } - this.topLogs = [...this.topLogsMap.values()].sort(LogData.sortComparerFn); this.keys = [...keysSet].sort(); + this.initSummary(summaryMap); } static isErrorLog(log: JSONLog): boolean { return ( log[LogData.logKeys.level] === LogData.levels.error || - !!log[LogData.logKeys.error] + !!log[LogData.logKeys.error] || + !!log[LogData.logKeys.Error] ); } @@ -79,33 +117,82 @@ class LogData { objectsUtils.getNestedKeys(log).forEach((k) => keysSet.add(k)); } - private initTopLogsMap(log: JSONLog) { - const msg = log[LogData.logKeys.msg]; - const cleanMsg = stringsUtils - .cleanText(msg) - .substring(0, LogData.msgCutOffLen) - .trim(); + private initSummary(summaryMap: SummaryMap) { + this.summary.msgs = [...summaryMap.msgs.values()].sort( + LogData.sortByLogsFn + ); + this.summary.httpCodes = [...summaryMap.httpCodes.values()].sort( + LogData.sortByMsgFn + ); + this.summary.jobs = [...summaryMap.jobs.values()].sort( + LogData.sortByLogsFn + ); + this.summary.plugins = [...summaryMap.plugins.values()].sort( + LogData.sortByLogsFn + ); + } - if (!this.topLogsMap.has(cleanMsg)) { - this.topLogsMap.set(cleanMsg, { - msg: cleanMsg, + private static initSummaryMap(log: JSONLog, summaryMap: SummaryMap) { + LogData.populateSummaryMap(log, summaryMap.msgs, LogData.msgKeySelector); + LogData.populateSummaryMap(log, summaryMap.jobs, LogData.jobKeySelector); + LogData.populateSummaryMap( + log, + summaryMap.httpCodes, + LogData.httpCodeKeySelector + ); + LogData.populateSummaryMap( + log, + summaryMap.plugins, + LogData.pluginKeySelector + ); + } + + private static populateSummaryMap( + log: JSONLog, + grpLogsMap: Map, + keySelectorFn: (log: JSONLog) => string | undefined + ) { + const key = keySelectorFn(log); + if (!key) return; + + if (!grpLogsMap.has(key)) { + grpLogsMap.set(key, { + msg: key, hasErrors: false, logs: [], }); } - const topLog = this.topLogsMap.get(cleanMsg)!; - topLog.logs.push(log); - if (!topLog.hasErrors && LogData.isErrorLog(log)) { - topLog.hasErrors = true; + const grpLog = grpLogsMap.get(key)!; + grpLog.logs.push(log); + if (!grpLog.hasErrors && LogData.isErrorLog(log)) { + grpLog.hasErrors = true; } } - private addLog(log: JSONLog) { - log[LogData.logKeys.fullData] = JSON.stringify(log); - this.logs.push(log); + private static msgKeySelector(log: JSONLog): string { + return LogData.getCutOffMsg(log); + } + + private static httpCodeKeySelector(log: JSONLog): string | undefined { + return log[LogData.logKeys.status_code] || log[LogData.logKeys.status]; + } + + private static jobKeySelector(log: JSONLog): string | undefined { + return log[LogData.logKeys.scheduler_name] || log[LogData.logKeys.worker]; + } + + private static pluginKeySelector(log: JSONLog): string | undefined { + return log[LogData.logKeys.plugin_id]; + } + + private static getCutOffMsg(log: JSONLog) { + return stringsUtils + .cleanText(log[LogData.logKeys.msg]) + .substring(0, LogData.msgCutOffLen) + .trim(); } } export default LogData; -export type { JSONLog, JSONLogs, GroupedMsg, LogsGenerator }; +export type { JSONLog, JSONLogs, GroupedMsg, LogsGenerator, Summary }; diff --git a/src/services/comparer.test.ts b/src/services/comparer.test.ts index 26d0114..777f302 100644 --- a/src/services/comparer.test.ts +++ b/src/services/comparer.test.ts @@ -1,5 +1,5 @@ import comparer from "./comparer"; -import LogData, { GroupedMsg } from "../models/logData"; +import LogData from "../models/logData"; describe("addLogData", () => { test("initial state", () => { @@ -11,11 +11,13 @@ describe("addLogData", () => { }); const logData1 = new LogData(); - logData1.topLogsMap = new Map([ - ["grp1", { logs: [], hasErrors: false, msg: "grp1" }], - ["grp2", { logs: [], hasErrors: false, msg: "grp2" }], - ["grp3", { logs: [], hasErrors: false, msg: "grp3" }], - ]); + logData1.summary = { + msgs: [ + { logs: [], hasErrors: false, msg: "grp1" }, + { logs: [], hasErrors: false, msg: "grp2" }, + { logs: [], hasErrors: false, msg: "grp3" }, + ], + } as any; test("1st logData", () => { comparer.addLogData(logData1); @@ -28,12 +30,14 @@ describe("addLogData", () => { }); const logData2 = new LogData(); - logData2.topLogsMap = new Map([ - ["grp11", { logs: [], hasErrors: false, msg: "grp11" }], - ["grp2", logData1.topLogsMap.get("grp2")!], - ["grp44", { logs: [], hasErrors: false, msg: "grp44" }], - ["grp3", logData1.topLogsMap.get("grp3")!], - ]); + logData2.summary = { + msgs: [ + { logs: [], hasErrors: false, msg: "grp11" }, + logData1.summary.msgs[1], + { logs: [], hasErrors: false, msg: "grp44" }, + logData1.summary.msgs[2], + ], + } as any; test("2nd logData", () => { comparer.addLogData(logData2); @@ -42,11 +46,8 @@ describe("addLogData", () => { expect(comparer.first(), "first").toEqual(logData1); expect(comparer.last(), "last").toEqual(logData2); - const added = [ - logData2.topLogsMap.get("grp11"), - logData2.topLogsMap.get("grp44"), - ]; - const removed = [logData1.topLogsMap.get("grp1")]; + const added = [logData2.summary.msgs[0], logData2.summary.msgs[2]]; + const removed = [logData1.summary.msgs[0]]; expect(comparer.added, "added").toEqual(added); expect(comparer.removed, "removed").toEqual(removed); }); diff --git a/src/services/comparer.ts b/src/services/comparer.ts index 061caf4..551121d 100644 --- a/src/services/comparer.ts +++ b/src/services/comparer.ts @@ -13,8 +13,8 @@ const added: GroupedMsg[] = []; const removed: GroupedMsg[] = []; function compare() { - const mapA = logDatas[0].topLogsMap; - const mapB = logDatas[1].topLogsMap; + const mapA = getLogMaps(logDatas[0].summary.msgs); + const mapB = getLogMaps(logDatas[1].summary.msgs); // Removed entries from 2nd log file for (const [k, v] of mapA) { @@ -30,8 +30,14 @@ function compare() { } } - added.sort(LogData.sortComparerFn); - removed.sort(LogData.sortComparerFn); + added.sort(LogData.sortByLogsFn); + removed.sort(LogData.sortByLogsFn); +} + +function getLogMaps(grpLogs: GroupedMsg[]): Map { + const map = new Map(); + grpLogs.forEach((l) => map.set(l.msg, l)); + return map; } const comparer = { diff --git a/src/services/normalizer.ts b/src/services/normalizer.ts index 5917d63..e7b0dca 100644 --- a/src/services/normalizer.ts +++ b/src/services/normalizer.ts @@ -4,7 +4,7 @@ import objectsUtils from "@al/utils/objects"; // Sample Log Line Format: // info [2023-10-16 02:24:52.930 +11:00] Received HTTP request caller="jobs/base_workers.go:97" worker=PostPersistentNotifications job_id=a6kay9tcptdymeezjng9i965mh dynamic1=value1 dynamic2=value2 -type ParserFunc = (logLine: string) => JSONLog | null; +type ParserFn = (logLine: string) => JSONLog | null; async function init(logData: LogData, file: File) { const text = await getText(file); @@ -12,14 +12,14 @@ async function init(logData: LogData, file: File) { const textSplitRegex = isJSON ? /\r?\n/ : /\r?\n(?=error|warn|info|verbose|debug|trace)/; - const parserFunc = isJSON ? jsonParser : plainParser; - logData.init(file, parse(text, textSplitRegex, parserFunc)); + const parserFn = isJSON ? jsonParser : plainParser; + logData.init(file, parse(text, textSplitRegex, parserFn)); } -function parse(text: string, textSplitRegex: RegExp, parserFunc: ParserFunc) { +function parse(text: string, textSplitRegex: RegExp, parserFn: ParserFn) { return function* (): LogsGenerator { for (const line of text.split(textSplitRegex)) { - yield parserFunc(line); + yield parserFn(line); } }; } @@ -75,58 +75,74 @@ function plainParser(logLine: string): JSONLog | null { function getText(file: File): Promise { return file.text(); - // Sample Test JSON Lines - // return Promise.resolve( - // [ - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "msg a", - // [LogData.logKeys.timestamp]: "2023-08-22 03:00:00.000 +10:00", - // }), - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "test b", - // [LogData.logKeys.timestamp]: "2023-08-22 03:05:00.000 +10:00", - // }), - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "msg c", - // [LogData.logKeys.timestamp]: "2023-08-22 03:10:00.000 +10:00", - // }), - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "test d", - // [LogData.logKeys.timestamp]: "2023-08-22 03:15:00.000 +10:00", - // }), - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "msg e", - // [LogData.logKeys.timestamp]: "2023-08-22 03:20:00.000 +10:00", - // }), - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "test f", - // [LogData.logKeys.timestamp]: "2023-08-22 03:30:00.000 +10:00", - // }), - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "msg g", - // [LogData.logKeys.timestamp]: "2023-08-22 03:35:00.000 +10:00", - // }), - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "test h", - // [LogData.logKeys.timestamp]: "2023-08-22 03:50:00.000 +10:00", - // }), - // JSON.stringify({ - // [LogData.logKeys.level]: "info", - // [LogData.logKeys.msg]: "msg i", - // [LogData.logKeys.timestamp]: "2023-08-22 03:55:00.000 +10:00", - // }), - // ].join("\n") - // ); + // return getTextSample(); } +// Sample Test JSON Lines +// function getTextSample(): Promise { +// return Promise.resolve( +// [ +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "msg a", +// [LogData.logKeys.timestamp]: "2023-08-22 03:00:00.000 +10:00", +// [LogData.logKeys.status_code]: "404", +// [LogData.logKeys.plugin_id]: "plugin-1", +// [LogData.logKeys.worker]: "job-1", +// }), +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "test b", +// [LogData.logKeys.timestamp]: "2023-08-22 03:05:00.000 +10:00", +// [LogData.logKeys.status_code]: "200", +// [LogData.logKeys.plugin_id]: "plugin-2", +// [LogData.logKeys.worker]: "job-2", +// }), +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "msg c", +// [LogData.logKeys.timestamp]: "2023-08-22 03:10:00.000 +10:00", +// [LogData.logKeys.status_code]: "404", +// [LogData.logKeys.worker]: "job-2", +// }), +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "test d", +// [LogData.logKeys.timestamp]: "2023-08-22 03:15:00.000 +10:00", +// [LogData.logKeys.status_code]: "200", +// [LogData.logKeys.plugin_id]: "plugin-2", +// }), +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "msg e", +// [LogData.logKeys.timestamp]: "2023-08-22 03:20:00.000 +10:00", +// [LogData.logKeys.plugin_id]: "plugin-2", +// [LogData.logKeys.worker]: "job-2", +// }), +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "test f", +// [LogData.logKeys.timestamp]: "2023-08-22 03:30:00.000 +10:00", +// }), +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "msg g", +// [LogData.logKeys.timestamp]: "2023-08-22 03:35:00.000 +10:00", +// }), +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "test h", +// [LogData.logKeys.timestamp]: "2023-08-22 03:50:00.000 +10:00", +// }), +// JSON.stringify({ +// [LogData.logKeys.level]: "info", +// [LogData.logKeys.msg]: "msg i", +// [LogData.logKeys.timestamp]: "2023-08-22 03:55:00.000 +10:00", +// }), +// ].join("\n") +// ); +// } + function isJSONLog(text: string): boolean { const firstLine = text.split(/\r?\n/, 1)[0]; const log = objectsUtils.parseJSON(firstLine);