From d1cd763a3de5676e2ceebd8106bea7bb5be17801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien?= Date: Fri, 6 Oct 2023 15:50:20 -0400 Subject: [PATCH 1/5] Added UI support for waitTillSegmentsLoad --- .../src/druid-models/execution/execution.ts | 28 +++++++++++++++++++ .../workbench-query/workbench-query.ts | 1 + .../helpers/execution/sql-task-execution.ts | 8 +++++- .../execution-progress-bar-pane.tsx | 1 + .../ingest-success-pane.tsx | 5 +++- 5 files changed, 41 insertions(+), 2 deletions(-) diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index 8f30711b2838..83199f94c318 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -182,6 +182,7 @@ export interface ExecutionValue { warnings?: ExecutionError[]; capacityInfo?: CapacityInfo; _payload?: MsqTaskPayloadResponse; + segmentStatus?: string; } export class Execution { @@ -292,6 +293,11 @@ export class Execution { const startTime = new Date(deepGet(taskReport, 'multiStageQuery.payload.status.startTime')); const durationMs = deepGet(taskReport, 'multiStageQuery.payload.status.durationMs'); + const segmentLoaderStatus = deepGet( + taskReport, + 'multiStageQuery.payload.status.segmentLoadWaiterStatus', + ); + let result: QueryResult | undefined; const resultsPayload: { signature: { name: string; type: string }[]; @@ -313,6 +319,7 @@ export class Execution { engine: 'sql-msq-task', id, status: Execution.normalizeTaskStatus(status), + segmentStatus: segmentLoaderStatus?.state, startTime: isNaN(startTime.getTime()) ? undefined : startTime, duration: typeof durationMs === 'number' ? durationMs : undefined, usageInfo: getUsageInfoFromStatusPayload( @@ -369,6 +376,7 @@ export class Execution { public readonly error?: ExecutionError; public readonly warnings?: ExecutionError[]; public readonly capacityInfo?: CapacityInfo; + public readonly segmentStatus?: string; public readonly _payload?: { payload: any; task: string }; @@ -390,6 +398,7 @@ export class Execution { this.error = value.error; this.warnings = nonEmptyArray(value.warnings) ? value.warnings : undefined; this.capacityInfo = value.capacityInfo; + this.segmentStatus = value.segmentStatus; this._payload = value._payload; } @@ -412,6 +421,7 @@ export class Execution { error: this.error, warnings: this.warnings, capacityInfo: this.capacityInfo, + segmentStatus: this.segmentStatus, _payload: this._payload, }; @@ -526,6 +536,24 @@ export class Execution { return status !== 'SUCCESS' && status !== 'FAILED'; } + public getSegmentStatusDescription() { + const { segmentStatus } = this; + + switch (segmentStatus) { + case 'INIT': + return 'Waiting for segments loading to start...'; + + case 'WAITING': + return 'Waiting for segments loading to complete...'; + + case 'SUCCESS': + return 'Segments loaded successfully'; + + default: + return ''; + } + } + public isFullyComplete(): boolean { if (this.isWaitingForQuery()) return false; diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index d7847287fa64..43fe3ea6528c 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -552,6 +552,7 @@ export class WorkbenchQuery { apiQuery.context.executionMode ??= 'async'; apiQuery.context.finalizeAggregations ??= !ingestQuery; apiQuery.context.groupByEnableMultiValueUnnesting ??= !ingestQuery; + apiQuery.context.waitTillSegmentsLoad ??= true; } if (Array.isArray(queryParameters) && queryParameters.length) { diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index d41370df16cf..ef4a077196ee 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -57,7 +57,13 @@ export interface SubmitTaskQueryOptions { export async function submitTaskQuery( options: SubmitTaskQueryOptions, ): Promise> { - const { query, context, prefixLines, cancelToken, preserveOnTermination, onSubmitted } = options; + const { query, prefixLines, cancelToken, preserveOnTermination, onSubmitted } = options; + + // setting waitTillSegmentsLoad to true by default + const context = { + waitTillSegmentsLoad: true, + ...(options.context || {}), + }; let sqlQuery: string; let jsonQuery: Record; diff --git a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx index 349bdaff93dd..fc6ace703175 100644 --- a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx +++ b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx @@ -78,6 +78,7 @@ export const ExecutionProgressBarPane = React.memo(function ExecutionProgressBar intent={stages ? Intent.PRIMARY : undefined} value={stages && execution.isWaitingForQuery() ? stages.overallProgress() : undefined} /> + {execution?.segmentStatus && } {stages && idx >= 0 && ( <> diff --git a/web-console/src/views/workbench-view/ingest-success-pane/ingest-success-pane.tsx b/web-console/src/views/workbench-view/ingest-success-pane/ingest-success-pane.tsx index 539ad234e7eb..04c89a1a0041 100644 --- a/web-console/src/views/workbench-view/ingest-success-pane/ingest-success-pane.tsx +++ b/web-console/src/views/workbench-view/ingest-success-pane/ingest-success-pane.tsx @@ -44,7 +44,8 @@ export const IngestSuccessPane = React.memo(function IngestSuccessPane( const warnings = execution.stages?.getWarningCount() || 0; - const duration = execution.duration; + const { duration, segmentStatus } = execution; + return (

@@ -67,6 +68,8 @@ export const IngestSuccessPane = React.memo(function IngestSuccessPane( Show details

+ + {segmentStatus &&

{execution.getSegmentStatusDescription()}

} {onQueryTab && (

Open new tab with:{' '} From 83fd09af2252fd91d78d92bda0241f9f1fbc9a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien?= Date: Mon, 9 Oct 2023 08:25:47 -0400 Subject: [PATCH 2/5] surfacing waitTillSegmentsLoad in the run panel --- .../druid-models/query-context/query-context.tsx | 16 ++++++++++++++++ .../views/workbench-view/run-panel/run-panel.tsx | 12 ++++++++++++ 2 files changed, 28 insertions(+) diff --git a/web-console/src/druid-models/query-context/query-context.tsx b/web-console/src/druid-models/query-context/query-context.tsx index d09fa73868ee..cdaf8dd84e35 100644 --- a/web-console/src/druid-models/query-context/query-context.tsx +++ b/web-console/src/druid-models/query-context/query-context.tsx @@ -162,6 +162,22 @@ export function changeFinalizeAggregations( : deepDelete(context, 'finalizeAggregations'); } +// waitTillSegmentsLoad + +export function getWaitTillSegmentsLoad(context: QueryContext): boolean | undefined { + const { waitTillSegmentsLoad } = context; + return typeof waitTillSegmentsLoad === 'boolean' ? waitTillSegmentsLoad : undefined; +} + +export function changeWaitTillSegmentsLoad( + context: QueryContext, + waitTillSegmentsLoad: boolean | undefined, +): QueryContext { + return typeof waitTillSegmentsLoad === 'boolean' + ? deepSet(context, 'waitTillSegmentsLoad', waitTillSegmentsLoad) + : deepDelete(context, 'waitTillSegmentsLoad'); +} + // groupByEnableMultiValueUnnesting export function getGroupByEnableMultiValueUnnesting(context: QueryContext): boolean | undefined { diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx index ec1b95ad38a4..6c976bbf4041 100644 --- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx +++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx @@ -45,6 +45,7 @@ import { changeUseApproximateCountDistinct, changeUseApproximateTopN, changeUseCache, + changeWaitTillSegmentsLoad, getDurableShuffleStorage, getFinalizeAggregations, getGroupByEnableMultiValueUnnesting, @@ -53,6 +54,7 @@ import { getUseApproximateCountDistinct, getUseApproximateTopN, getUseCache, + getWaitTillSegmentsLoad, summarizeIndexSpec, } from '../../../druid-models'; import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils'; @@ -110,6 +112,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const maxParseExceptions = getMaxParseExceptions(queryContext); const finalizeAggregations = getFinalizeAggregations(queryContext); + const waitTillSegmentsLoad = getWaitTillSegmentsLoad(queryContext); const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext); const sqlJoinAlgorithm = queryContext.sqlJoinAlgorithm ?? 'broadcast'; const selectDestination = queryContext.selectDestination ?? 'taskReport'; @@ -311,6 +314,15 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { changeQueryContext(changeFinalizeAggregations(queryContext, v)) } /> + + changeQueryContext(changeWaitTillSegmentsLoad(queryContext, v)) + } + /> Date: Mon, 9 Oct 2023 09:52:13 -0400 Subject: [PATCH 3/5] fixed snapshots --- web-console/src/druid-models/execution/execution.spec.ts | 3 +++ .../__snapshots__/execution-details-pane.spec.tsx.snap | 3 +++ 2 files changed, 6 insertions(+) diff --git a/web-console/src/druid-models/execution/execution.spec.ts b/web-console/src/druid-models/execution/execution.spec.ts index a6b55db68085..fac129baf9b6 100644 --- a/web-console/src/druid-models/execution/execution.spec.ts +++ b/web-console/src/druid-models/execution/execution.spec.ts @@ -268,6 +268,7 @@ describe('Execution', () => { "maxNumTasks": 2, }, "result": undefined, + "segmentStatus": undefined, "sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", @@ -643,6 +644,7 @@ describe('Execution', () => { "sqlQuery": undefined, "sqlQueryId": undefined, }, + "segmentStatus": undefined, "sqlQuery": undefined, "stages": undefined, "startTime": 2023-07-05T21:33:19.147Z, @@ -679,6 +681,7 @@ describe('Execution', () => { "nativeQuery": undefined, "queryContext": undefined, "result": undefined, + "segmentStatus": undefined, "sqlQuery": undefined, "stages": undefined, "startTime": 2023-07-05T21:40:39.986Z, diff --git a/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap index 6f3c23bee00d..f88a9a3a98be 100644 --- a/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap @@ -286,6 +286,7 @@ PARTITIONED BY DAY", "maxParseExceptions": 2, }, "result": undefined, + "segmentStatus": undefined, "sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", @@ -909,6 +910,7 @@ PARTITIONED BY DAY", "maxParseExceptions": 2, }, "result": undefined, + "segmentStatus": undefined, "sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", @@ -1576,6 +1578,7 @@ PARTITIONED BY DAY", "maxParseExceptions": 2, }, "result": undefined, + "segmentStatus": undefined, "sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", From 7ca3405bdb885957f08acd6aa555a4396d66f6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien?= Date: Tue, 10 Oct 2023 08:47:18 -0400 Subject: [PATCH 4/5] More details and no extraneous SQL query for segments unless needed --- .../src/druid-models/execution/execution.ts | 38 +++++++++++++++---- .../helpers/execution/sql-task-execution.ts | 5 +++ .../execution-details-pane.tsx | 30 ++++++++++++++- .../execution-progress-bar-pane.tsx | 5 ++- .../ingest-success-pane.tsx | 5 ++- 5 files changed, 70 insertions(+), 13 deletions(-) diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index 83199f94c318..1899f1260724 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -164,6 +164,18 @@ function formatPendingMessage( } } +interface SegmentStatus { + duration: number; + onDemandSegments: number; + pendingSegments: number; + precachedSegments: number; + startTime: Date; + state: 'INIT' | 'WAITING' | 'SUCCESS'; + totalSegments: number; + unknownSegments: number; + usedSegments: number; +} + export interface ExecutionValue { engine: DruidEngine; id: string; @@ -182,7 +194,7 @@ export interface ExecutionValue { warnings?: ExecutionError[]; capacityInfo?: CapacityInfo; _payload?: MsqTaskPayloadResponse; - segmentStatus?: string; + segmentStatus?: SegmentStatus; } export class Execution { @@ -319,7 +331,7 @@ export class Execution { engine: 'sql-msq-task', id, status: Execution.normalizeTaskStatus(status), - segmentStatus: segmentLoaderStatus?.state, + segmentStatus: segmentLoaderStatus, startTime: isNaN(startTime.getTime()) ? undefined : startTime, duration: typeof durationMs === 'number' ? durationMs : undefined, usageInfo: getUsageInfoFromStatusPayload( @@ -376,7 +388,7 @@ export class Execution { public readonly error?: ExecutionError; public readonly warnings?: ExecutionError[]; public readonly capacityInfo?: CapacityInfo; - public readonly segmentStatus?: string; + public readonly segmentStatus?: SegmentStatus; public readonly _payload?: { payload: any; task: string }; @@ -539,19 +551,29 @@ export class Execution { public getSegmentStatusDescription() { const { segmentStatus } = this; - switch (segmentStatus) { + let label = ''; + + switch (segmentStatus?.state) { case 'INIT': - return 'Waiting for segments loading to start...'; + label = 'Waiting for segments loading to start...'; + break; case 'WAITING': - return 'Waiting for segments loading to complete...'; + label = 'Waiting for segments loading to complete...'; + break; case 'SUCCESS': - return 'Segments loaded successfully'; + label = 'Segments loaded successfully in ' + segmentStatus.duration + 'ms.'; + break; default: - return ''; + break; } + + return { + label, + ...segmentStatus, + }; } public isFullyComplete(): boolean { diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index ef4a077196ee..75b82d17b868 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -267,6 +267,11 @@ export async function updateExecutionWithDatasourceLoadedIfNeeded( return execution; } + // This means we don't have to perform the SQL query to check if the segments are loaded + if (execution.queryContext?.waitTillSegmentsLoad === true) { + return execution.markDestinationDatasourceLoaded(); + } + const endTime = execution.getEndTime(); if ( !endTime || // If endTime is not set (this is not expected to happen) then just bow out diff --git a/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx b/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx index 72a6350d7ba1..0f1200a8661f 100644 --- a/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx +++ b/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx @@ -23,7 +23,7 @@ import React, { useState } from 'react'; import { FancyTabPane } from '../../../components'; import type { Execution } from '../../../druid-models'; -import { pluralIfNeeded } from '../../../utils'; +import { formatDuration, formatDurationWithMs, pluralIfNeeded } from '../../../utils'; import { DestinationPagesPane } from '../destination-pages-pane/destination-pages-pane'; import { ExecutionErrorPane } from '../execution-error-pane/execution-error-pane'; import { ExecutionStagesPane } from '../execution-stages-pane/execution-stages-pane'; @@ -40,7 +40,8 @@ export type ExecutionDetailsTab = | 'result' | 'pages' | 'error' - | 'warnings'; + | 'warnings' + | 'segmentStatus'; interface ExecutionDetailsPaneProps { execution: Execution; @@ -53,6 +54,7 @@ export const ExecutionDetailsPane = React.memo(function ExecutionDetailsPane( ) { const { execution, initTab, goToTask } = props; const [activeTab, setActiveTab] = useState(initTab || 'general'); + const segmentStatusDescription = execution.getSegmentStatusDescription(); function renderContent() { switch (activeTab) { @@ -120,6 +122,25 @@ export const ExecutionDetailsPane = React.memo(function ExecutionDetailsPane( case 'warnings': return ; + case 'segmentStatus': + return ( + <> +

+ Duration:{' '} + {segmentStatusDescription.duration + ? formatDurationWithMs(segmentStatusDescription.duration) + : '-'} + {execution.duration + ? ` (query duration was ${formatDuration(execution.duration)})` + : ''} +

+

Total segments: {segmentStatusDescription.totalSegments ?? '-'}

+

Used segments: {segmentStatusDescription.usedSegments ?? '-'}

+

Precached segments: {segmentStatusDescription.precachedSegments ?? '-'}

+

On demand segments: {segmentStatusDescription.onDemandSegments ?? '-'}

+ + ); + default: return; } @@ -146,6 +167,11 @@ export const ExecutionDetailsPane = React.memo(function ExecutionDetailsPane( label: 'Native query', icon: IconNames.COG, }, + Boolean(execution.segmentStatus) && { + id: 'segmentStatus', + label: 'Segments', + icon: IconNames.HEAT_GRID, + }, execution.result && { id: 'result', label: 'Results', diff --git a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx index fc6ace703175..4520c20a39b1 100644 --- a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx +++ b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx @@ -50,6 +50,9 @@ export const ExecutionProgressBarPane = React.memo(function ExecutionProgressBar const idx = stages ? stages.currentStageIndex() : -1; const waitingForSegments = stages && !execution.isWaitingForQuery(); + + const segmentStatusDescription = execution?.getSegmentStatusDescription(); + return (