Skip to content

Commit

Permalink
Stats info (#10)
Browse files Browse the repository at this point in the history
* add first and last occurrence info to stats

* fix search case-sensitivity
  • Loading branch information
vish9812 authored Feb 27, 2024
1 parent 48d1495 commit 8dc7596
Show file tree
Hide file tree
Showing 7 changed files with 184 additions and 97 deletions.
11 changes: 8 additions & 3 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,14 @@ tasks:
- cmd:install

test:
desc: run tests
deps:
- ui:test
desc: run tests
deps:
- ui:test

run:
desc: run the UI
deps:
- ui:run

build:
desc: transpile, bundle and create the release zip
Expand Down
94 changes: 68 additions & 26 deletions cmd/src/commands/summary/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,36 @@ import { Table } from "console-table-printer";
import { parseArgs } from "util";
import { cpus } from "node:os";
import type { ICmd } from "@al/cmd/utils/cmd-runner";
import { type ITask, type IResult } from "./worker";
import type {
ITask,
IResult,
IGroupedMsg,
ISummaryMap,
ISummaryMapGeneric,
} from "./worker";
import WorkerPool from "@al/cmd/utils/worker-pool";
import fileHelper from "@al/cmd/utils/file-helper";
import type {
GroupedMsg,
Summary as SummaryData,
SummaryMap,
} from "@al/ui/models/logData";
import LogData from "@al/ui/models/logData";

let workerURL = new URL("worker.ts", import.meta.url);

interface IStats extends Omit<IResult, "filePath"> {
interface IStats extends Omit<IResult, "filePath" | "dataMap"> {
minTimeFile: string;
maxTimeFile: string;
dataMap: IStatsSummaryMap;
}

interface IStatsGroupedMsg extends IGroupedMsg {
firstFile: string;
lastFile: string;
}

type IStatsSummaryMap = ISummaryMapGeneric<IStatsGroupedMsg>;

interface ISummary {
msgs: IStatsGroupedMsg[];
httpCodes: IStatsGroupedMsg[];
jobs: IStatsGroupedMsg[];
plugins: IStatsGroupedMsg[];
}

const stats: IStats = {
Expand All @@ -27,10 +42,10 @@ const stats: IStats = {
minTimeFile: "",
size: 0,
dataMap: {
httpCodes: new Map<string, GroupedMsg>(),
jobs: new Map<string, GroupedMsg>(),
msgs: new Map<string, GroupedMsg>(),
plugins: new Map<string, GroupedMsg>(),
httpCodes: new Map<string, IStatsGroupedMsg>(),
jobs: new Map<string, IStatsGroupedMsg>(),
msgs: new Map<string, IStatsGroupedMsg>(),
plugins: new Map<string, IStatsGroupedMsg>(),
},
};

Expand Down Expand Up @@ -109,7 +124,7 @@ async function processLogs() {
await readFiles(filePaths);
console.log("=========End Read Files=========");

const summary = LogData.initSummary(stats.dataMap);
const summary = initSummary(stats.dataMap);

writeContent(summary);
}
Expand Down Expand Up @@ -155,37 +170,64 @@ function processFileResponse(fileStats: IResult) {

stats.size += fileStats.size;

initSummaryMap(fileStats.dataMap);
initSummaryMap(fileStats.dataMap, fileStats.filePath);
}

function initSummaryMap(dataMap: SummaryMap) {
mergeIntoOverallMap(dataMap.httpCodes, stats.dataMap.httpCodes);
mergeIntoOverallMap(dataMap.jobs, stats.dataMap.jobs);
mergeIntoOverallMap(dataMap.msgs, stats.dataMap.msgs);
mergeIntoOverallMap(dataMap.plugins, stats.dataMap.plugins);
function initSummaryMap(dataMap: ISummaryMap, filePath: string) {
mergeIntoOverallMap(dataMap.httpCodes, stats.dataMap.httpCodes, filePath);
mergeIntoOverallMap(dataMap.jobs, stats.dataMap.jobs, filePath);
mergeIntoOverallMap(dataMap.msgs, stats.dataMap.msgs, filePath);
mergeIntoOverallMap(dataMap.plugins, stats.dataMap.plugins, filePath);
}

function mergeIntoOverallMap(
fileMap: Map<string, GroupedMsg>,
overallMap: Map<string, GroupedMsg>
fileMap: Map<string, IGroupedMsg>,
overallMap: Map<string, IStatsGroupedMsg>,
filePath: string
) {
for (const [k, v] of fileMap) {
if (!overallMap.has(k)) {
overallMap.set(k, {
msg: v.msg,
hasErrors: false,
logs: [],
logsCount: 0,
firstTime: v.firstTime,
lastTime: v.lastTime,
firstFile: filePath,
lastFile: filePath,
});
}

const grpOverall = overallMap.get(k)!;
grpOverall.hasErrors = grpOverall.hasErrors || v.hasErrors;
grpOverall.hasErrors ||= v.hasErrors;
grpOverall.logsCount += v.logsCount!;

if (v.firstTime < grpOverall.firstTime) {
grpOverall.firstTime = v.firstTime;
grpOverall.firstFile = filePath;
}

if (v.lastTime > grpOverall.lastTime) {
grpOverall.lastTime = v.lastTime;
grpOverall.lastFile = filePath;
}
}
}

function writeContent(summary: SummaryData) {
function summarySorterFn(a: IStatsGroupedMsg, b: IStatsGroupedMsg) {
return b.logsCount - a.logsCount;
}

function initSummary(summaryMap: IStatsSummaryMap): ISummary {
return {
msgs: [...summaryMap.msgs.values()].sort(summarySorterFn),
httpCodes: [...summaryMap.httpCodes.values()].sort(summarySorterFn),
jobs: [...summaryMap.jobs.values()].sort(summarySorterFn),
plugins: [...summaryMap.plugins.values()].sort(summarySorterFn),
};
}

function writeContent(summary: ISummary) {
console.log();

stats.size = prettyBytes(stats.size) as any;
Expand Down Expand Up @@ -217,15 +259,15 @@ function writeContent(summary: SummaryData) {
writeGroupedMsgs(summary.plugins, "Plugins");
}

function writeGroupedMsgs(grpMsgs: GroupedMsg[], title: string) {
function writeGroupedMsgs(grpMsgs: IStatsGroupedMsg[], title: string) {
console.log();

const table = new Table({
columns: [
{ name: "msg", title: title, alignment: "left" },
{ name: "logsCount", title: "Count" },
],
disabledColumns: ["hasErrors", "logs"],
disabledColumns: ["hasErrors"],
});

for (const grp of grpMsgs.slice(0, flags.topLogsCount)) {
Expand Down
92 changes: 75 additions & 17 deletions cmd/src/commands/summary/worker.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,24 @@
import { parentPort } from "node:worker_threads";
import LogData, {
type GroupedMsg,
type SummaryMap,
} from "@al/ui/models/logData";
import LogData, { type JSONLog } from "@al/ui/models/logData";
import normalizer from "@al/ui/services/normalizer";

interface IGroupedMsg {
msg: string;
logsCount: number;
hasErrors: boolean;
firstTime: string;
lastTime: string;
}

interface ISummaryMapGeneric<TGroupedMsg> {
msgs: Map<string, TGroupedMsg>;
httpCodes: Map<string, TGroupedMsg>;
jobs: Map<string, TGroupedMsg>;
plugins: Map<string, TGroupedMsg>;
}

type ISummaryMap = ISummaryMapGeneric<IGroupedMsg>;

interface ITask {
filePath: string;
}
Expand All @@ -14,34 +28,36 @@ interface IResult {
minTime: string;
maxTime: string;
size: number;
dataMap: SummaryMap;
dataMap: ISummaryMap;
}

// Thread Code
if (parentPort) {
parentPort.on("message", async (task: ITask) => {
const stats = await processFile(task);
parentPort!.postMessage(stats);
console.log(`Processed File: ${task.filePath}`);
console.log(`Processed File: ${stats.filePath}`);
});
}

async function processFile(task: ITask): Promise<IResult> {
const stats: IResult = {
filePath: task.filePath,
filePath: "",
maxTime: "0",
minTime: "z",
size: 0,
dataMap: {
httpCodes: new Map<string, GroupedMsg>(),
jobs: new Map<string, GroupedMsg>(),
msgs: new Map<string, GroupedMsg>(),
plugins: new Map<string, GroupedMsg>(),
httpCodes: new Map<string, IGroupedMsg>(),
jobs: new Map<string, IGroupedMsg>(),
msgs: new Map<string, IGroupedMsg>(),
plugins: new Map<string, IGroupedMsg>(),
},
};

const logFile = Bun.file(task.filePath);
stats.size = logFile.size;
// Keep only the last folder from the filePath
stats.filePath = task.filePath.split("/").slice(-2).join("/");

const text = await logFile.text();

Expand All @@ -54,18 +70,60 @@ async function processFile(task: ITask): Promise<IResult> {
for (const jsonLog of logsGeneratorFn()) {
if (!jsonLog) continue;

if (jsonLog[LogData.logKeys.timestamp] > stats.maxTime) {
stats.maxTime = jsonLog[LogData.logKeys.timestamp];
const time = jsonLog[LogData.logKeys.timestamp];

if (time < stats.minTime) {
stats.minTime = time;
}

if (jsonLog[LogData.logKeys.timestamp] < stats.minTime) {
stats.minTime = jsonLog[LogData.logKeys.timestamp];
if (time > stats.maxTime) {
stats.maxTime = time;
}

LogData.initSummaryMap(jsonLog, stats.dataMap, false);
initSummaryMap(jsonLog, stats.dataMap);
}

return stats;
}

export type { ITask, IResult };
function initSummaryMap(log: JSONLog, summaryMap: ISummaryMap) {
populateSummaryMap(log, summaryMap.msgs, LogData.msgKeySelector);
populateSummaryMap(log, summaryMap.jobs, LogData.jobKeySelector);
populateSummaryMap(log, summaryMap.httpCodes, LogData.httpCodeKeySelector);
populateSummaryMap(log, summaryMap.plugins, LogData.pluginKeySelector);
}

function populateSummaryMap(
log: JSONLog,
grpLogsMap: Map<string, IGroupedMsg>,
keySelectorFn: (log: JSONLog) => string | undefined
) {
const key = keySelectorFn(log);
if (!key) return;

const time = log[LogData.logKeys.timestamp];

if (!grpLogsMap.has(key)) {
grpLogsMap.set(key, {
msg: key,
hasErrors: false,
logsCount: 0,
firstTime: time,
lastTime: time,
});
}

const grpLog = grpLogsMap.get(key)!;
grpLog.logsCount++;
grpLog.hasErrors ||= LogData.isErrorLog(log);

if (time < grpLog.firstTime) {
grpLog.firstTime = time;
}

if (time > grpLog.lastTime) {
grpLog.lastTime = time;
}
}

export type { ITask, IResult, IGroupedMsg, ISummaryMap, ISummaryMapGeneric };
5 changes: 5 additions & 0 deletions ui/Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,8 @@ tasks:
cmds:
- pnpm build
- cp analog.sh analog.ps1 analog

run:
desc: run
cmds:
- pnpm start
4 changes: 3 additions & 1 deletion ui/src/components/filters/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,9 @@ function Filters(props: FiltersProps) {
<TextField
label="Search"
value={term.value}
onChange={(_, val) => setFilters("terms", i(), "value", val)}
onChange={(_, val) =>
setFilters("terms", i(), "value", val.toLowerCase())
}
onKeyDown={handleEnterKey}
/>
</>
Expand Down
Loading

0 comments on commit 8dc7596

Please sign in to comment.