Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added UI support for waitTillSegmentsLoad #15110

Merged
merged 5 commits into from
Oct 11, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions web-console/src/druid-models/execution/execution.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,7 @@ describe('Execution', () => {
"maxNumTasks": 2,
},
"result": undefined,
"segmentStatus": undefined,
"sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL
SELECT
TIME_PARSE(\\"timestamp\\") AS \\"__time\\",
Expand Down Expand Up @@ -643,6 +644,7 @@ describe('Execution', () => {
"sqlQuery": undefined,
"sqlQueryId": undefined,
},
"segmentStatus": undefined,
"sqlQuery": undefined,
"stages": undefined,
"startTime": 2023-07-05T21:33:19.147Z,
Expand Down Expand Up @@ -679,6 +681,7 @@ describe('Execution', () => {
"nativeQuery": undefined,
"queryContext": undefined,
"result": undefined,
"segmentStatus": undefined,
"sqlQuery": undefined,
"stages": undefined,
"startTime": 2023-07-05T21:40:39.986Z,
Expand Down
28 changes: 28 additions & 0 deletions web-console/src/druid-models/execution/execution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ export interface ExecutionValue {
warnings?: ExecutionError[];
capacityInfo?: CapacityInfo;
_payload?: MsqTaskPayloadResponse;
segmentStatus?: string;
}

export class Execution {
Expand Down Expand Up @@ -292,6 +293,11 @@ export class Execution {
const startTime = new Date(deepGet(taskReport, 'multiStageQuery.payload.status.startTime'));
const durationMs = deepGet(taskReport, 'multiStageQuery.payload.status.durationMs');

const segmentLoaderStatus = deepGet(
taskReport,
'multiStageQuery.payload.status.segmentLoadWaiterStatus',
);

let result: QueryResult | undefined;
const resultsPayload: {
signature: { name: string; type: string }[];
Expand All @@ -313,6 +319,7 @@ export class Execution {
engine: 'sql-msq-task',
id,
status: Execution.normalizeTaskStatus(status),
segmentStatus: segmentLoaderStatus?.state,
startTime: isNaN(startTime.getTime()) ? undefined : startTime,
duration: typeof durationMs === 'number' ? durationMs : undefined,
usageInfo: getUsageInfoFromStatusPayload(
Expand Down Expand Up @@ -369,6 +376,7 @@ export class Execution {
public readonly error?: ExecutionError;
public readonly warnings?: ExecutionError[];
public readonly capacityInfo?: CapacityInfo;
public readonly segmentStatus?: string;

public readonly _payload?: { payload: any; task: string };

Expand All @@ -390,6 +398,7 @@ export class Execution {
this.error = value.error;
this.warnings = nonEmptyArray(value.warnings) ? value.warnings : undefined;
this.capacityInfo = value.capacityInfo;
this.segmentStatus = value.segmentStatus;

this._payload = value._payload;
}
Expand All @@ -412,6 +421,7 @@ export class Execution {
error: this.error,
warnings: this.warnings,
capacityInfo: this.capacityInfo,
segmentStatus: this.segmentStatus,

_payload: this._payload,
};
Expand Down Expand Up @@ -526,6 +536,24 @@ export class Execution {
return status !== 'SUCCESS' && status !== 'FAILED';
}

public getSegmentStatusDescription() {
const { segmentStatus } = this;

switch (segmentStatus) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this also display the segments breakdown after ingestion? This information should be available in the same place

"totalSegments": 1,
"usedSegments": 1,
"precachedSegments": 0,
"onDemandSegments": 0,

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should also show the time for the query part and ingestion part separately, so that any user can know the actual time taken for the query and won't mistake it as druid being slow. This information should also be available in segmentLoadStatus and its parent object.

case 'INIT':
return 'Waiting for segments loading to start...';

case 'WAITING':
return 'Waiting for segments loading to complete...';

case 'SUCCESS':
return 'Segments loaded successfully';

default:
return '';
}
}

public isFullyComplete(): boolean {
if (this.isWaitingForQuery()) return false;

Expand Down
16 changes: 16 additions & 0 deletions web-console/src/druid-models/query-context/query-context.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,22 @@ export function changeFinalizeAggregations(
: deepDelete(context, 'finalizeAggregations');
}

// waitTillSegmentsLoad

export function getWaitTillSegmentsLoad(context: QueryContext): boolean | undefined {
const { waitTillSegmentsLoad } = context;
return typeof waitTillSegmentsLoad === 'boolean' ? waitTillSegmentsLoad : undefined;
}

export function changeWaitTillSegmentsLoad(
context: QueryContext,
waitTillSegmentsLoad: boolean | undefined,
): QueryContext {
return typeof waitTillSegmentsLoad === 'boolean'
? deepSet(context, 'waitTillSegmentsLoad', waitTillSegmentsLoad)
: deepDelete(context, 'waitTillSegmentsLoad');
}

// groupByEnableMultiValueUnnesting

export function getGroupByEnableMultiValueUnnesting(context: QueryContext): boolean | undefined {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,7 @@ export class WorkbenchQuery {
apiQuery.context.executionMode ??= 'async';
apiQuery.context.finalizeAggregations ??= !ingestQuery;
apiQuery.context.groupByEnableMultiValueUnnesting ??= !ingestQuery;
apiQuery.context.waitTillSegmentsLoad ??= true;
}

if (Array.isArray(queryParameters) && queryParameters.length) {
Expand Down
8 changes: 7 additions & 1 deletion web-console/src/helpers/execution/sql-task-execution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,13 @@ export interface SubmitTaskQueryOptions {
export async function submitTaskQuery(
options: SubmitTaskQueryOptions,
): Promise<Execution | IntermediateQueryState<Execution>> {
const { query, context, prefixLines, cancelToken, preserveOnTermination, onSubmitted } = options;
const { query, prefixLines, cancelToken, preserveOnTermination, onSubmitted } = options;

// setting waitTillSegmentsLoad to true by default
const context = {
waitTillSegmentsLoad: true,
...(options.context || {}),
};

let sqlQuery: string;
let jsonQuery: Record<string, any>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,7 @@ PARTITIONED BY DAY",
"maxParseExceptions": 2,
},
"result": undefined,
"segmentStatus": undefined,
"sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL
SELECT
TIME_PARSE(\\"timestamp\\") AS \\"__time\\",
Expand Down Expand Up @@ -909,6 +910,7 @@ PARTITIONED BY DAY",
"maxParseExceptions": 2,
},
"result": undefined,
"segmentStatus": undefined,
"sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL
SELECT
TIME_PARSE(\\"timestamp\\") AS \\"__time\\",
Expand Down Expand Up @@ -1576,6 +1578,7 @@ PARTITIONED BY DAY",
"maxParseExceptions": 2,
},
"result": undefined,
"segmentStatus": undefined,
"sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL
SELECT
TIME_PARSE(\\"timestamp\\") AS \\"__time\\",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ export const ExecutionProgressBarPane = React.memo(function ExecutionProgressBar
intent={stages ? Intent.PRIMARY : undefined}
value={stages && execution.isWaitingForQuery() ? stages.overallProgress() : undefined}
/>
{execution?.segmentStatus && <Label>{execution.getSegmentStatusDescription()}</Label>}
{stages && idx >= 0 && (
<>
<Label>{`Current stage (${idx + 1} of ${stages.stageCount()})`}</Label>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ export const IngestSuccessPane = React.memo(function IngestSuccessPane(

const warnings = execution.stages?.getWarningCount() || 0;

const duration = execution.duration;
const { duration, segmentStatus } = execution;

return (
<div className="ingest-success-pane">
<p>
Expand All @@ -67,6 +68,8 @@ export const IngestSuccessPane = React.memo(function IngestSuccessPane(
Show details
</span>
</p>

{segmentStatus && <p>{execution.getSegmentStatusDescription()}</p>}
{onQueryTab && (
<p>
Open new tab with:{' '}
Expand Down
12 changes: 12 additions & 0 deletions web-console/src/views/workbench-view/run-panel/run-panel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import {
changeUseApproximateCountDistinct,
changeUseApproximateTopN,
changeUseCache,
changeWaitTillSegmentsLoad,
getDurableShuffleStorage,
getFinalizeAggregations,
getGroupByEnableMultiValueUnnesting,
Expand All @@ -53,6 +54,7 @@ import {
getUseApproximateCountDistinct,
getUseApproximateTopN,
getUseCache,
getWaitTillSegmentsLoad,
summarizeIndexSpec,
} from '../../../druid-models';
import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils';
Expand Down Expand Up @@ -110,6 +112,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {

const maxParseExceptions = getMaxParseExceptions(queryContext);
const finalizeAggregations = getFinalizeAggregations(queryContext);
const waitTillSegmentsLoad = getWaitTillSegmentsLoad(queryContext);
const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext);
const sqlJoinAlgorithm = queryContext.sqlJoinAlgorithm ?? 'broadcast';
const selectDestination = queryContext.selectDestination ?? 'taskReport';
Expand Down Expand Up @@ -311,6 +314,15 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
changeQueryContext(changeFinalizeAggregations(queryContext, v))
}
/>
<MenuTristate
icon={IconNames.STOPWATCH}
text="Wait until segments have loaded"
value={waitTillSegmentsLoad}
undefinedEffectiveValue /* ={true} */
onValueChange={v =>
changeQueryContext(changeWaitTillSegmentsLoad(queryContext, v))
}
/>
<MenuTristate
icon={IconNames.FORK}
text="Enable GroupBy multi-value unnesting"
Expand Down
Loading