From 4e2383baadee6c4a17d977aab65a0caa3ca1abc0 Mon Sep 17 00:00:00 2001 From: Adarsh Sanjeev Date: Wed, 25 Oct 2023 18:08:37 +0530 Subject: [PATCH] Rename segment load wait parameter (#15251) --- docs/multi-stage-query/reference.md | 3 ++- .../druid/msq/util/MultiStageQueryContext.java | 2 +- .../query-context/query-context.tsx | 18 +++++++++--------- .../workbench-query/workbench-query.spec.ts | 2 +- .../workbench-query/workbench-query.ts | 2 +- .../helpers/execution/sql-task-execution.ts | 6 +++--- .../workbench-view/run-panel/run-panel.tsx | 10 +++++----- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/docs/multi-stage-query/reference.md b/docs/multi-stage-query/reference.md index 9ec50de0a9b1..3fd2335d0525 100644 --- a/docs/multi-stage-query/reference.md +++ b/docs/multi-stage-query/reference.md @@ -246,9 +246,10 @@ The following table lists the context parameters for the MSQ task engine: | `durableShuffleStorage` | SELECT, INSERT, REPLACE

Whether to use durable storage for shuffle mesh. To use this feature, configure the durable storage at the server level using `druid.msq.intermediate.storage.enable=true`). If these properties are not configured, any query with the context variable `durableShuffleStorage=true` fails with a configuration error.

| `false` | | `faultTolerance` | SELECT, INSERT, REPLACE

Whether to turn on fault tolerance mode or not. Failed workers are retried based on [Limits](#limits). Cannot be used when `durableShuffleStorage` is explicitly set to false. | `false` | | `selectDestination` | SELECT

Controls where the final result of the select query is written.
Use `taskReport`(the default) to write select results to the task report. This is not scalable since task reports size explodes for large results
Use `durableStorage` to write results to durable storage location. For large results sets, its recommended to use `durableStorage` . To configure durable storage see [`this`](#durable-storage) section. | `taskReport` | -| `waitTillSegmentsLoad` | INSERT, REPLACE

If set, the ingest query waits for the generated segment to be loaded before exiting, else the ingest query exits without waiting. The task and live reports contain the information about the status of loading segments if this flag is set. This will ensure that any future queries made after the ingestion exits will include results from the ingestion. The drawback is that the controller task will stall till the segments are loaded. | `false` | +| `waitUntilSegmentsLoad` | INSERT, REPLACE

If set, the ingest query waits for the generated segment to be loaded before exiting, else the ingest query exits without waiting. The task and live reports contain the information about the status of loading segments if this flag is set. This will ensure that any future queries made after the ingestion exits will include results from the ingestion. The drawback is that the controller task will stall until the segments are loaded. | `false` | | `includeSegmentSource` | SELECT, INSERT, REPLACE

Controls the sources, which will be queried for results in addition to the segments present on deep storage. Can be `NONE` or `REALTIME`. If this value is `NONE`, only non-realtime (published and used) segments will be downloaded from deep storage. If this value is `REALTIME`, results will also be included from realtime tasks. | `NONE` | | `rowsPerPage` | SELECT

The number of rows per page to target. The actual number of rows per page may be somewhat higher or lower than this number. In most cases, use the default.
This property comes into effect only when `selectDestination` is set to `durableStorage` | 100000 | + ## Joins Joins in multi-stage queries use one of two algorithms based on what you set the [context parameter](#context-parameters) `sqlJoinAlgorithm` to: diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java index f6caa6da0590..80027ea112e5 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/util/MultiStageQueryContext.java @@ -114,7 +114,7 @@ public class MultiStageQueryContext public static final String CTX_FAULT_TOLERANCE = "faultTolerance"; public static final boolean DEFAULT_FAULT_TOLERANCE = false; - public static final String CTX_SEGMENT_LOAD_WAIT = "waitTillSegmentsLoad"; + public static final String CTX_SEGMENT_LOAD_WAIT = "waitUntilSegmentsLoad"; public static final boolean DEFAULT_SEGMENT_LOAD_WAIT = false; public static final String CTX_MAX_INPUT_BYTES_PER_WORKER = "maxInputBytesPerWorker"; diff --git a/web-console/src/druid-models/query-context/query-context.tsx b/web-console/src/druid-models/query-context/query-context.tsx index cdaf8dd84e35..c0445038a60f 100644 --- a/web-console/src/druid-models/query-context/query-context.tsx +++ b/web-console/src/druid-models/query-context/query-context.tsx @@ -162,20 +162,20 @@ export function changeFinalizeAggregations( : deepDelete(context, 'finalizeAggregations'); } -// waitTillSegmentsLoad +// waitUntilSegmentsLoad -export function getWaitTillSegmentsLoad(context: QueryContext): boolean | undefined { - const { waitTillSegmentsLoad } = context; - return typeof waitTillSegmentsLoad === 'boolean' ? waitTillSegmentsLoad : undefined; +export function getWaitUntilSegmentsLoad(context: QueryContext): boolean | undefined { + const { waitUntilSegmentsLoad } = context; + return typeof waitUntilSegmentsLoad === 'boolean' ? waitUntilSegmentsLoad : undefined; } -export function changeWaitTillSegmentsLoad( +export function changeWaitUntilSegmentsLoad( context: QueryContext, - waitTillSegmentsLoad: boolean | undefined, + waitUntilSegmentsLoad: boolean | undefined, ): QueryContext { - return typeof waitTillSegmentsLoad === 'boolean' - ? deepSet(context, 'waitTillSegmentsLoad', waitTillSegmentsLoad) - : deepDelete(context, 'waitTillSegmentsLoad'); + return typeof waitUntilSegmentsLoad === 'boolean' + ? deepSet(context, 'waitUntilSegmentsLoad', waitUntilSegmentsLoad) + : deepDelete(context, 'waitUntilSegmentsLoad'); } // groupByEnableMultiValueUnnesting diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index 079ccde5a2dd..78ac3e8f1e74 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -423,7 +423,7 @@ describe('WorkbenchQuery', () => { finalizeAggregations: false, groupByEnableMultiValueUnnesting: false, useCache: false, - waitTillSegmentsLoad: true, + waitUntilSegmentsLoad: true, }, header: true, query: 'INSERT INTO wiki2 SELECT * FROM wikipedia', diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index 43fe3ea6528c..621e5028f4b3 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -552,7 +552,7 @@ export class WorkbenchQuery { apiQuery.context.executionMode ??= 'async'; apiQuery.context.finalizeAggregations ??= !ingestQuery; apiQuery.context.groupByEnableMultiValueUnnesting ??= !ingestQuery; - apiQuery.context.waitTillSegmentsLoad ??= true; + apiQuery.context.waitUntilSegmentsLoad ??= true; } if (Array.isArray(queryParameters) && queryParameters.length) { diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index 75b82d17b868..f690886ad05d 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -59,9 +59,9 @@ export async function submitTaskQuery( ): Promise> { const { query, prefixLines, cancelToken, preserveOnTermination, onSubmitted } = options; - // setting waitTillSegmentsLoad to true by default + // setting waitUntilSegmentsLoad to true by default const context = { - waitTillSegmentsLoad: true, + waitUntilSegmentsLoad: true, ...(options.context || {}), }; @@ -268,7 +268,7 @@ export async function updateExecutionWithDatasourceLoadedIfNeeded( } // This means we don't have to perform the SQL query to check if the segments are loaded - if (execution.queryContext?.waitTillSegmentsLoad === true) { + if (execution.queryContext?.waitUntilSegmentsLoad === true) { return execution.markDestinationDatasourceLoaded(); } diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx index 6c976bbf4041..0bcd0a3d8983 100644 --- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx +++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx @@ -45,7 +45,7 @@ import { changeUseApproximateCountDistinct, changeUseApproximateTopN, changeUseCache, - changeWaitTillSegmentsLoad, + changeWaitUntilSegmentsLoad, getDurableShuffleStorage, getFinalizeAggregations, getGroupByEnableMultiValueUnnesting, @@ -54,7 +54,7 @@ import { getUseApproximateCountDistinct, getUseApproximateTopN, getUseCache, - getWaitTillSegmentsLoad, + getWaitUntilSegmentsLoad, summarizeIndexSpec, } from '../../../druid-models'; import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils'; @@ -112,7 +112,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { const maxParseExceptions = getMaxParseExceptions(queryContext); const finalizeAggregations = getFinalizeAggregations(queryContext); - const waitTillSegmentsLoad = getWaitTillSegmentsLoad(queryContext); + const waitUntilSegmentsLoad = getWaitUntilSegmentsLoad(queryContext); const groupByEnableMultiValueUnnesting = getGroupByEnableMultiValueUnnesting(queryContext); const sqlJoinAlgorithm = queryContext.sqlJoinAlgorithm ?? 'broadcast'; const selectDestination = queryContext.selectDestination ?? 'taskReport'; @@ -317,10 +317,10 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) { - changeQueryContext(changeWaitTillSegmentsLoad(queryContext, v)) + changeQueryContext(changeWaitUntilSegmentsLoad(queryContext, v)) } />