From 35cc7c8d9072e4a7b81aabec0835e3ff86a7ba87 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 27 Sep 2024 10:46:32 +0100 Subject: [PATCH] HPCC-32689 Include/Exclude pending scopes in metrics Signed-off-by: Gordon Smith --- esp/src/src-react/components/ECLArchive.tsx | 2 +- esp/src/src-react/components/Metrics.tsx | 242 +++++------------- .../src-react/components/MetricsOptions.tsx | 2 - .../src-react/components/MetricsScopes.tsx | 124 +++++++++ esp/src/src-react/hooks/metrics.ts | 23 +- esp/src/src-react/util/metricGraph.ts | 2 +- esp/src/src/nls/hpcc.ts | 1 + 7 files changed, 210 insertions(+), 186 deletions(-) create mode 100644 esp/src/src-react/components/MetricsScopes.tsx diff --git a/esp/src/src-react/components/ECLArchive.tsx b/esp/src/src-react/components/ECLArchive.tsx index a84ceaae3ac..69558a5fe48 100644 --- a/esp/src/src-react/components/ECLArchive.tsx +++ b/esp/src/src-react/components/ECLArchive.tsx @@ -39,7 +39,7 @@ export const ECLArchive: React.FunctionComponent = ({ const [fullscreen, setFullscreen] = React.useState(false); const [dockpanel, setDockpanel] = React.useState(); const [_archiveXmlStr, _workunit2, _state2, archive, refreshArchive] = useWorkunitArchive(wuid); - const [metrics, _columns, _activities, _properties, _measures, _scopeTypes, _fetchStatus, refreshMetrics] = useWorkunitMetrics(wuid, scopeFilterDefault, nestedFilterDefault); + const { metrics, refresh: refreshMetrics } = useWorkunitMetrics(wuid, scopeFilterDefault, nestedFilterDefault); const [markers, setMarkers] = React.useState<{ lineNum: number, label: string }[]>([]); const [selectionText, setSelectionText] = React.useState(""); const [selectedMetrics, setSelectedMetrics] = React.useState([]); diff --git a/esp/src/src-react/components/Metrics.tsx b/esp/src/src-react/components/Metrics.tsx index f6fec060b8e..0cb9a7aff41 100644 --- a/esp/src/src-react/components/Metrics.tsx +++ b/esp/src/src-react/components/Metrics.tsx @@ -3,10 +3,9 @@ import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, IIconProps, S import { Label, Spinner, ToggleButton } from "@fluentui/react-components"; import { typographyStyles } from "@fluentui/react-theme"; import { useConst } from "@fluentui/react-hooks"; -import { bundleIcon, Folder20Filled, Folder20Regular, FolderOpen20Filled, FolderOpen20Regular, TextCaseTitleRegular, TextCaseTitleFilled } from "@fluentui/react-icons"; -import { Database } from "@hpcc-js/common"; -import { WorkunitsServiceEx, IScope, splitMetric } from "@hpcc-js/comms"; -import { CellFormatter, ColumnFormat, ColumnType, DBStore, RowType, Table } from "@hpcc-js/dgrid"; +import { bundleIcon, Folder20Filled, Folder20Regular, FolderOpen20Filled, FolderOpen20Regular, TextCaseTitleRegular, TextCaseTitleFilled, BranchForkHintRegular, BranchForkFilled } from "@fluentui/react-icons"; +import { WorkunitsServiceEx, IScope } from "@hpcc-js/comms"; +import { Table } from "@hpcc-js/dgrid"; import { scopedLogger } from "@hpcc-js/util"; import nlsHPCC from "src/nlsHPCC"; import { WUTimelineNoFetch } from "src/Timings"; @@ -24,6 +23,7 @@ import { MetricsOptions } from "./MetricsOptions"; import { BreadcrumbInfo, OverflowBreadcrumb } from "./controls/OverflowBreadcrumb"; import { MetricsPropertiesTables } from "./MetricsPropertiesTables"; import { MetricsSQL } from "./MetricsSQL"; +import { ScopesTable } from "./MetricsScopes"; const logger = scopedLogger("src-react/components/Metrics.tsx"); @@ -36,126 +36,6 @@ const defaultUIState = { hasSelection: false }; -class ColumnFormatEx extends ColumnFormat { - formatterFunc(): CellFormatter | undefined { - const colIdx = this._owner.columns().indexOf("__StdDevs"); - - return function (this: ColumnType, cell: any, row: RowType): string { - return row[colIdx]; - }; - } -} - -class DBStoreEx extends DBStore { - - constructor(protected _table: TableEx, db: Database.Grid) { - super(db); - } - - sort(opts) { - this._table.sort(opts); - return this; - } -} - -class TableEx extends Table { - - constructor() { - super(); - this._store = new DBStoreEx(this, this._db); - } - - scopeFilterFunc(row: object, scopeFilter: string, matchCase: boolean): boolean { - const filter = scopeFilter.trim(); - if (filter) { - let field = ""; - const colonIdx = filter.indexOf(":"); - if (colonIdx > 0) { - field = filter.substring(0, colonIdx); - } - if (field) { - const value: string = !matchCase ? row[field]?.toString().toLowerCase() : row[field]?.toString(); - const filterValue: string = !matchCase ? filter.toLowerCase() : filter; - return value?.indexOf(filterValue.substring(colonIdx + 1)) >= 0 ?? false; - } - for (const field in row) { - const value: string = !matchCase ? row[field].toString().toLowerCase() : row[field].toString(); - const filterValue: string = !matchCase ? filter.toLowerCase() : filter; - return value?.indexOf(filterValue) >= 0 ?? false; - } - return false; - } - return true; - } - - _rawDataMap: { [id: number]: string } = {}; - metrics(metrics: any[], scopeTypes: string[], properties: string[], scopeFilter: string, matchCase: boolean): this { - this - .columns(["##"]) // Reset hash to force recalculation of default widths - .columns(["##", nlsHPCC.Type, "StdDevs", nlsHPCC.Scope, ...properties, "__StdDevs"]) - .columnFormats([ - new ColumnFormatEx() - .column("StdDevs") - .paletteID("StdDevs") - .min(0) - .max(6), - new ColumnFormat() - .column("__StdDevs") - .width(0) - ]) - .data(metrics - .filter(m => this.scopeFilterFunc(m, scopeFilter, matchCase)) - .filter(row => { - return scopeTypes.indexOf(row.type) >= 0; - }).map((row, idx) => { - if (idx === 0) { - this._rawDataMap = { - 0: "##", 1: "type", 2: "__StdDevs", 3: "name" - }; - properties.forEach((p, idx2) => { - this._rawDataMap[4 + idx2] = p; - }); - } - row.__hpcc_id = row.name; - return [idx, row.type, row.__StdDevs === 0 ? undefined : row.__StdDevs, row.name, ...properties.map(p => { - return row.__groupedProps[p]?.Value ?? - row.__groupedProps[p]?.Max ?? - row.__groupedProps[p]?.Avg ?? - row.__formattedProps[p] ?? - row[p] ?? - ""; - }), row.__StdDevs === 0 ? "" : row.__StdDevsSource, row]; - })) - ; - return this; - } - - sort(opts) { - const optsEx = opts.map(opt => { - return { - idx: opt.property, - metricLabel: this._rawDataMap[opt.property], - splitMetricLabel: splitMetric(this._rawDataMap[opt.property]), - descending: opt.descending - }; - }); - - const lparamIdx = this.columns().length; - this._db.data().sort((l, r) => { - const llparam = l[lparamIdx]; - const rlparam = r[lparamIdx]; - for (const { idx, metricLabel, splitMetricLabel, descending } of optsEx) { - const lval = llparam[metricLabel] ?? llparam[`${splitMetricLabel.measure}Max${splitMetricLabel.label}`] ?? llparam[`${splitMetricLabel.measure}Avg${splitMetricLabel.label}`] ?? l[idx]; - const rval = rlparam[metricLabel] ?? rlparam[`${splitMetricLabel.measure}Max${splitMetricLabel.label}`] ?? rlparam[`${splitMetricLabel.measure}Avg${splitMetricLabel.label}`] ?? r[idx]; - if ((lval === undefined && rval !== undefined) || lval < rval) return descending ? 1 : -1; - if ((lval !== undefined && rval === undefined) || lval > rval) return descending ? -1 : 1; - } - return 0; - }); - return this; - } -} - type SelectedMetricsSource = "" | "scopesTable" | "scopesSqlTable" | "metricGraphWidget" | "hotspot" | "reset"; const TIMELINE_FIXEDHEIGHT = 152; @@ -178,7 +58,7 @@ export const Metrics: React.FunctionComponent = ({ const [selectedMetricsSource, setSelectedMetricsSource] = React.useState(""); const [selectedMetrics, setSelectedMetrics] = React.useState([]); const [selectedMetricsPtr, setSelectedMetricsPtr] = React.useState(-1); - const [metrics, columns, _activities, _properties, _measures, _scopeTypes, fetchStatus, refresh] = useWUQueryMetrics(wuid, querySet, queryId); + const { metrics, columns, status, refresh } = useWUQueryMetrics(wuid, querySet, queryId); const { viewIds, viewId, setViewId, view, updateView } = useMetricsViews(); const [showMetricOptions, setShowMetricOptions] = React.useState(false); const [dockpanel, setDockpanel] = React.useState(); @@ -190,6 +70,7 @@ export const Metrics: React.FunctionComponent = ({ const [isLayoutComplete, setIsLayoutComplete] = React.useState(false); const [isRenderComplete, setIsRenderComplete] = React.useState(false); const [dot, setDot] = React.useState(""); + const [includePendingItems, setIncludePendingItems] = React.useState(false); const [matchCase, setMatchCase] = React.useState(false); React.useEffect(() => { @@ -249,53 +130,6 @@ export const Metrics: React.FunctionComponent = ({ } }, [metrics, timeline, view.showTimeline]); - // Scopes Table --- - const onChangeScopeFilter = React.useCallback((event: React.FormEvent, newValue?: string) => { - setScopeFilter(newValue || ""); - }, []); - - const scopesSelectionChanged = React.useCallback((source: SelectedMetricsSource, selection: IScope[]) => { - setSelectedMetricsSource(source); - pushUrl(`${parentUrl}/${selection.map(row => row.__lparam?.id ?? row.id).join(",")}`); - }, [parentUrl]); - - const scopesTable = useConst(() => new TableEx() - .multiSelect(true) - .metrics([], view.scopeTypes, view.properties, scopeFilter, matchCase) - .sortable(true) - ); - - React.useEffect(() => { - scopesTable - .on("click", debounce((row, col, sel) => { - if (sel) { - scopesSelectionChanged("scopesTable", scopesTable.selection()); - } - }), true) - ; - }, [scopesSelectionChanged, scopesTable]); - - React.useEffect(() => { - scopesTable - .metrics(metrics, view.scopeTypes, view.properties, scopeFilter, matchCase) - .lazyRender() - ; - }, [matchCase, metrics, scopeFilter, scopesTable, view.properties, view.scopeTypes]); - - const updateScopesTable = React.useCallback((selection: IScope[]) => { - if (scopesTable?.renderCount() > 0 && selectedMetricsSource !== "scopesTable") { - scopesTable.selection([]); - if (selection.length) { - const selRows = scopesTable.data().filter(row => { - return selection.indexOf(row[row.length - 1]) >= 0; - }); - scopesTable.render(() => { - scopesTable.selection(selRows); - }); - } - } - }, [scopesTable, selectedMetricsSource]); - // Graph --- const metricGraph = useConst(() => new MetricGraph()); const metricGraphWidget = useConst(() => new MetricGraphWidget() @@ -438,7 +272,7 @@ export const Metrics: React.FunctionComponent = ({ ], [metricGraphWidget, selectedMetrics.length, trackSelection]); const spinnerLabel: string = React.useMemo((): string => { - if (fetchStatus === FetchStatus.STARTED) { + if (status === FetchStatus.STARTED) { return nlsHPCC.FetchingData; } else if (!isLayoutComplete) { return `${nlsHPCC.PerformingLayout}(${dot.split("\n").length})`; @@ -446,7 +280,7 @@ export const Metrics: React.FunctionComponent = ({ return nlsHPCC.RenderSVG; } return ""; - }, [fetchStatus, isLayoutComplete, isRenderComplete, dot]); + }, [status, isLayoutComplete, isRenderComplete, dot]); const breadcrumbs = React.useMemo(() => { return lineage.map(item => { @@ -460,6 +294,63 @@ export const Metrics: React.FunctionComponent = ({ }); }, [lineage, selectedLineage]); + // Scopes Table --- + const onChangeScopeFilter = React.useCallback((event: React.FormEvent, newValue?: string) => { + setScopeFilter(newValue || ""); + }, []); + + const scopesSelectionChanged = React.useCallback((source: SelectedMetricsSource, selection: IScope[]) => { + setSelectedMetricsSource(source); + pushUrl(`${parentUrl}/${selection.map(row => row.__lparam?.id ?? row.id).join(",")}`); + }, [parentUrl]); + + const scopesTable = useConst(() => new ScopesTable() + .multiSelect(true) + .metrics([], view.scopeTypes, view.properties, scopeFilter, matchCase) + .sortable(true) + ); + + React.useEffect(() => { + scopesTable + .on("click", debounce((row, col, sel) => { + if (sel) { + scopesSelectionChanged("scopesTable", scopesTable.selection()); + } + }), true) + ; + }, [scopesSelectionChanged, scopesTable]); + + React.useEffect(() => { + const scopesTableMetrics = includePendingItems ? metrics : metrics.filter(row => { + if (metricGraph.isVertex(row)) { + return metricGraph.vertexStatus(row) !== "unknown"; + } else if (metricGraph.isEdge(row)) { + return metricGraph.edgeStatus(row) !== "unknown"; + } else if (metricGraph.isSubgraph(row)) { + return metricGraph.subgraphStatus(row) !== "unknown"; + } + return true; + }); + scopesTable + .metrics(scopesTableMetrics, view.scopeTypes, view.properties, scopeFilter, matchCase) + .lazyRender() + ; + }, [includePendingItems, matchCase, metricGraph, metrics, scopeFilter, scopesTable, view.properties, view.scopeTypes]); + + const updateScopesTable = React.useCallback((selection: IScope[]) => { + if (scopesTable?.renderCount() > 0 && selectedMetricsSource !== "scopesTable") { + scopesTable.selection([]); + if (selection.length) { + const selRows = scopesTable.data().filter(row => { + return selection.indexOf(row[row.length - 1]) >= 0; + }); + scopesTable.render(() => { + scopesTable.selection(selRows); + }); + } + } + }, [scopesTable, selectedMetricsSource]); + // Props Table --- const crossTabTable = useConst(() => new Table() .columns([nlsHPCC.Property, nlsHPCC.Value]) @@ -637,8 +528,6 @@ export const Metrics: React.FunctionComponent = ({ setShowMetricOptions(show); }, []); - console.log("View ID", viewId, view.scopeTypes); - return @@ -650,6 +539,7 @@ export const Metrics: React.FunctionComponent = ({ + : } title={nlsHPCC.IncludePendingItems} checked={includePendingItems} onClick={() => { setIncludePendingItems(!includePendingItems); }} /> diff --git a/esp/src/src-react/components/MetricsOptions.tsx b/esp/src/src-react/components/MetricsOptions.tsx index aa0d20a108f..7aded51a910 100644 --- a/esp/src/src-react/components/MetricsOptions.tsx +++ b/esp/src/src-react/components/MetricsOptions.tsx @@ -157,8 +157,6 @@ export const MetricsOptions: React.FunctionComponent = ({ } }, [addView, dirtyView, view]); - console.log("dirtyView.scopeTypes", viewId, view.scopeTypes); - return <> diff --git a/esp/src/src-react/components/MetricsScopes.tsx b/esp/src/src-react/components/MetricsScopes.tsx new file mode 100644 index 00000000000..cc8d611e8f3 --- /dev/null +++ b/esp/src/src-react/components/MetricsScopes.tsx @@ -0,0 +1,124 @@ +import nlsHPCC from "src/nlsHPCC"; +import { Database } from "@hpcc-js/common"; +import { splitMetric, IScope } from "@hpcc-js/comms"; +import { CellFormatter, ColumnFormat, ColumnType, DBStore, RowType, Table } from "@hpcc-js/dgrid"; + +class ColumnFormatEx extends ColumnFormat { + formatterFunc(): CellFormatter | undefined { + const colIdx = this._owner.columns().indexOf("__StdDevs"); + + return function (this: ColumnType, cell: any, row: RowType): string { + return row[colIdx]; + }; + } +} + +class DBStoreEx extends DBStore { + + constructor(protected _table: ScopesTable, db: Database.Grid) { + super(db); + } + + sort(opts) { + this._table.sort(opts); + return this; + } +} + +export class ScopesTable extends Table { + + constructor() { + super(); + this._store = new DBStoreEx(this, this._db); + } + + scopeFilterFunc(row: IScope, scopeFilter: string, matchCase: boolean): boolean { + const filter = scopeFilter.trim(); + if (filter) { + let field = ""; + const colonIdx = filter.indexOf(":"); + if (colonIdx > 0) { + field = filter.substring(0, colonIdx); + } + if (field) { + const value: string = !matchCase ? row[field]?.toString().toLowerCase() : row[field]?.toString(); + const filterValue: string = !matchCase ? filter.toLowerCase() : filter; + return value?.indexOf(filterValue.substring(colonIdx + 1)) >= 0 ?? false; + } + for (const field in row) { + const value: string = !matchCase ? row[field].toString().toLowerCase() : row[field].toString(); + const filterValue: string = !matchCase ? filter.toLowerCase() : filter; + return value?.indexOf(filterValue) >= 0 ?? false; + } + return false; + } + return true; + } + + _rawDataMap: { [id: number]: string } = {}; + metrics(metrics: IScope[], scopeTypes: string[], properties: string[], scopeFilter: string, matchCase: boolean): this { + this + .columns(["##"]) // Reset hash to force recalculation of default widths + .columns(["##", nlsHPCC.Type, "StdDevs", nlsHPCC.Scope, ...properties, "__StdDevs"]) + .columnFormats([ + new ColumnFormatEx() + .column("StdDevs") + .paletteID("StdDevs") + .min(0) + .max(6), + new ColumnFormat() + .column("__StdDevs") + .width(0) + ]) + .data(metrics + .filter(m => this.scopeFilterFunc(m, scopeFilter, matchCase)) + .filter(row => { + return scopeTypes.indexOf(row.type) >= 0; + }).map((row, idx) => { + if (idx === 0) { + this._rawDataMap = { + 0: "##", 1: "type", 2: "__StdDevs", 3: "name" + }; + properties.forEach((p, idx2) => { + this._rawDataMap[4 + idx2] = p; + }); + } + row.__hpcc_id = row.name; + return [idx, row.type, row.__StdDevs === 0 ? undefined : row.__StdDevs, row.name, ...properties.map(p => { + return row.__groupedProps[p]?.Value ?? + row.__groupedProps[p]?.Max ?? + row.__groupedProps[p]?.Avg ?? + row.__formattedProps[p] ?? + row[p] ?? + ""; + }), row.__StdDevs === 0 ? "" : row.__StdDevsSource, row]; + })) + ; + return this; + } + + sort(opts) { + const optsEx = opts.map(opt => { + return { + idx: opt.property, + metricLabel: this._rawDataMap[opt.property], + splitMetricLabel: splitMetric(this._rawDataMap[opt.property]), + descending: opt.descending + }; + }); + + const lparamIdx = this.columns().length; + this._db.data().sort((l, r) => { + const llparam = l[lparamIdx]; + const rlparam = r[lparamIdx]; + for (const { idx, metricLabel, splitMetricLabel, descending } of optsEx) { + const lval = llparam[metricLabel] ?? llparam[`${splitMetricLabel.measure}Max${splitMetricLabel.label}`] ?? llparam[`${splitMetricLabel.measure}Avg${splitMetricLabel.label}`] ?? l[idx]; + const rval = rlparam[metricLabel] ?? rlparam[`${splitMetricLabel.measure}Max${splitMetricLabel.label}`] ?? rlparam[`${splitMetricLabel.measure}Avg${splitMetricLabel.label}`] ?? r[idx]; + if ((lval === undefined && rval !== undefined) || lval < rval) return descending ? 1 : -1; + if ((lval !== undefined && rval === undefined) || lval > rval) return descending ? -1 : 1; + } + return 0; + }); + return this; + } +} diff --git a/esp/src/src-react/hooks/metrics.ts b/esp/src/src-react/hooks/metrics.ts index fb08eb2d731..2b0a47687f2 100644 --- a/esp/src/src-react/hooks/metrics.ts +++ b/esp/src/src-react/hooks/metrics.ts @@ -246,11 +246,22 @@ const nestedFilterDefault: WsWorkunits.NestedFilter = { ScopeTypes: [] }; +export interface useMetricsResult { + metrics: IScope[]; + columns: { [id: string]: any }; + activities: WsWorkunits.Activity2[]; + properties: WsWorkunits.Property2[]; + measures: string[]; + scopeTypes: string[]; + status: FetchStatus; + refresh: () => void; +} + export function useWorkunitMetrics( wuid: string, scopeFilter: Partial = scopeFilterDefault, nestedFilter: WsWorkunits.NestedFilter = nestedFilterDefault -): [IScope[], { [id: string]: any }, WsWorkunits.Activity2[], WsWorkunits.Property2[], string[], string[], FetchStatus, () => void] { +): useMetricsResult { const [workunit, state] = useWorkunit(wuid); const [data, setData] = React.useState([]); @@ -303,7 +314,7 @@ export function useWorkunitMetrics( }); }, [workunit, state, count, scopeFilter, nestedFilter]); - return [data, columns, activities, properties, measures, scopeTypes, status, increment]; + return { metrics: data, columns, activities, properties, measures, scopeTypes, status, refresh: increment }; } export function useQueryMetrics( @@ -311,7 +322,7 @@ export function useQueryMetrics( queryId: string, scopeFilter: Partial = scopeFilterDefault, nestedFilter: WsWorkunits.NestedFilter = nestedFilterDefault -): [IScope[], { [id: string]: any }, WsWorkunits.Activity2[], WsWorkunits.Property2[], string[], string[], FetchStatus, () => void] { +): useMetricsResult { const [query, state, _refresh] = useQuery(querySet, queryId); const [data, setData] = React.useState([]); @@ -364,7 +375,7 @@ export function useQueryMetrics( }); }, [query, state, count, scopeFilter, nestedFilter]); - return [data, columns, activities, properties, measures, scopeTypes, status, increment]; + return { metrics: data, columns, activities, properties, measures, scopeTypes, status, refresh: increment }; } export function useWUQueryMetrics( @@ -373,8 +384,8 @@ export function useWUQueryMetrics( queryId: string, scopeFilter: Partial = scopeFilterDefault, nestedFilter: WsWorkunits.NestedFilter = nestedFilterDefault -): [IScope[], { [id: string]: any }, WsWorkunits.Activity2[], WsWorkunits.Property2[], string[], string[], FetchStatus, () => void] { +): useMetricsResult { const wuMetrics = useWorkunitMetrics(wuid, scopeFilter, nestedFilter); const queryMetrics = useQueryMetrics(querySet, queryId, scopeFilter, nestedFilter); - return querySet && queryId ? [...queryMetrics] : [...wuMetrics]; + return querySet && queryId ? { ...queryMetrics } : { ...wuMetrics }; } diff --git a/esp/src/src-react/util/metricGraph.ts b/esp/src/src-react/util/metricGraph.ts index 5322f65aac6..0d46d99290b 100644 --- a/esp/src/src-react/util/metricGraph.ts +++ b/esp/src/src-react/util/metricGraph.ts @@ -164,7 +164,7 @@ export class MetricGraph extends Graph2 { return retVal.reverse(); } - load(data: any[]): this { + load(data: IScope[]): this { this.clear(); // Index all items --- diff --git a/esp/src/src/nls/hpcc.ts b/esp/src/src/nls/hpcc.ts index 300feb3aaec..6c87a0415a7 100644 --- a/esp/src/src/nls/hpcc.ts +++ b/esp/src/src/nls/hpcc.ts @@ -411,6 +411,7 @@ export = { IgnoreGlobalStoreOutEdges: "Ignore Global Store Out Edges", Import: "Import", Inactive: "Inactive", + IncludePendingItems: "Include pending items", IncludeSlaveLogs: "Include worker logs", IncludeSubFileInfo: "Include sub file info?", Index: "Index",