From 263e106714d989ee6a462577185c3c591de1d4fd Mon Sep 17 00:00:00 2001 From: 317brian <53799971+317brian@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:52:51 -0700 Subject: [PATCH 1/6] docs: remove experimental note from unnest docs (#15123) * docs: remove experimental note from unnest docs * remove flag needed to use unnest --- docs/querying/datasource.md | 5 ----- docs/querying/sql.md | 6 ------ docs/tutorials/tutorial-unnest-arrays.md | 12 ------------ 3 files changed, 23 deletions(-) diff --git a/docs/querying/datasource.md b/docs/querying/datasource.md index ae87431587a1..5853f1e6dfc9 100644 --- a/docs/querying/datasource.md +++ b/docs/querying/datasource.md @@ -401,11 +401,6 @@ future versions: ### `unnest` -:::info - The unnest datasource is [experimental](../development/experimental.md). Its API and behavior are subject - to change in future releases. It is not recommended to use this feature in production at this time. -::: - Use the `unnest` datasource to unnest a column with multiple values in an array. For example, you have a source column that looks like this: diff --git a/docs/querying/sql.md b/docs/querying/sql.md index 13259bdf4044..952c7c419a27 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -87,11 +87,6 @@ documentation. ## UNNEST -:::info - The UNNEST SQL function is [experimental](../development/experimental.md). Its API and behavior are subject - to change in future releases. It is not recommended to use this feature in production at this time. -::: - The UNNEST clause unnests array values. It's the SQL equivalent to the [unnest datasource](./datasource.md#unnest). The source for UNNEST can be an array or an input that's been transformed into an array, such as with helper functions like MV_TO_ARRAY or ARRAY. The following is the general syntax for UNNEST, specifically a query that returns the column that gets unnested: @@ -110,7 +105,6 @@ SELECT column_alias_name FROM datasource CROSS JOIN UNNEST(source_expression1) A Keep the following things in mind when writing your query: -- You must include the context parameter `"enableUnnest": true`. - You can unnest multiple source expressions in a single query. - Notice the CROSS JOIN between the datasource and the UNNEST function. This is needed in most cases of the UNNEST function. Specifically, it is not needed when you're unnesting an inline array since the array itself is the datasource. - If you view the native explanation of a SQL UNNEST, you'll notice that Druid uses `j0.unnest` as a virtual column to perform the unnest. An underscore is added for each unnest, so you may notice virtual columns named `_j0.unnest` or `__j0.unnest`. diff --git a/docs/tutorials/tutorial-unnest-arrays.md b/docs/tutorials/tutorial-unnest-arrays.md index 49fdfe98af25..86b5407e0e7c 100644 --- a/docs/tutorials/tutorial-unnest-arrays.md +++ b/docs/tutorials/tutorial-unnest-arrays.md @@ -30,11 +30,6 @@ import TabItem from '@theme/TabItem'; If you're looking for information about how to unnest `COMPLEX` columns, see [Nested columns](../querying/nested-columns.md). ::: -:::info - The unnest datasource and UNNEST SQL function are [experimental](../development/experimental.md). Their API and behavior are subject - to change in future releases. It is not recommended to use this feature in production at this time. -::: - This tutorial demonstrates how to use the unnest datasource to unnest a column that has data stored in arrays. For example, if you have a column named `dim3` with values like `[a,b]` or `[c,d,f]`, the unnest datasource can output the data to a new column with individual rows that contain single values like `a` and `b`. When doing this, be mindful of the following: - Unnesting data can dramatically increase the total number of rows. @@ -166,12 +161,6 @@ The following is the general syntax for UNNEST: SELECT column_alias_name FROM datasource CROSS JOIN UNNEST(source_expression) AS table_alias_name(column_alias_name) ``` -In addition, you must supply the following context parameter: - -```json -"enableUnnest": "true" -``` - For more information about the syntax, see [UNNEST](../querying/sql.md#unnest). ### Unnest a single source expression in a datasource @@ -645,7 +634,6 @@ The following Scan query unnests the column `dim3` into `d3` and a virtual colum ], "legacy": false, "context": { - "enableUnnest": "true", "queryId": "2618b9ce-6c0d-414e-b88d-16fb59b9c481", "sqlOuterLimit": 1001, "sqlQueryId": "2618b9ce-6c0d-414e-b88d-16fb59b9c481", From 5f860724561d2e9c1c3219bc40fc41c487984efa Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 11 Oct 2023 10:33:45 +0530 Subject: [PATCH 2/6] Prepare master for Druid 29 (#15121) Prepare master for Druid 29 --- benchmarks/pom.xml | 2 +- cloud/aws-common/pom.xml | 2 +- cloud/gcp-common/pom.xml | 2 +- distribution/docker/docker-compose.yml | 10 +++++----- distribution/pom.xml | 2 +- extensions-contrib/aliyun-oss-extensions/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/compressed-bigdecimal/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/dropwizard-emitter/pom.xml | 2 +- extensions-contrib/druid-iceberg-extensions/pom.xml | 2 +- extensions-contrib/gce-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/influx-extensions/pom.xml | 2 +- extensions-contrib/influxdb-emitter/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- .../kubernetes-overlord-extensions/pom.xml | 2 +- .../materialized-view-maintenance/pom.xml | 2 +- extensions-contrib/materialized-view-selection/pom.xml | 2 +- extensions-contrib/momentsketch/pom.xml | 2 +- extensions-contrib/moving-average-query/pom.xml | 2 +- extensions-contrib/opentelemetry-emitter/pom.xml | 2 +- extensions-contrib/opentsdb-emitter/pom.xml | 2 +- extensions-contrib/prometheus-emitter/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/tdigestsketch/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/azure-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-aws-rds-extensions/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-bloom-filter/pom.xml | 2 +- extensions-core/druid-catalog/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/druid-pac4j/pom.xml | 2 +- extensions-core/druid-ranger-security/pom.xml | 2 +- extensions-core/ec2-extensions/pom.xml | 2 +- extensions-core/google-extensions/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/kinesis-indexing-service/pom.xml | 2 +- extensions-core/kubernetes-extensions/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/multi-stage-query/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/orc-extensions/pom.xml | 2 +- extensions-core/parquet-extensions/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- extensions-core/testing-tools/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests-ex/cases/pom.xml | 2 +- integration-tests-ex/image/pom.xml | 2 +- integration-tests-ex/tools/pom.xml | 2 +- integration-tests/pom.xml | 2 +- pom.xml | 2 +- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- web-console/package-lock.json | 4 ++-- web-console/package.json | 2 +- web-console/pom.xml | 2 +- web-console/unified-console.html | 2 +- 78 files changed, 83 insertions(+), 83 deletions(-) diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index fdada96d98a6..416d18c7f442 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/cloud/aws-common/pom.xml b/cloud/aws-common/pom.xml index 4bad5332ceef..50bf6d92bf13 100644 --- a/cloud/aws-common/pom.xml +++ b/cloud/aws-common/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/cloud/gcp-common/pom.xml b/cloud/gcp-common/pom.xml index 14179c4095be..31e421033790 100644 --- a/cloud/gcp-common/pom.xml +++ b/cloud/gcp-common/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml index 9813a00f6a58..bafb39394594 100644 --- a/distribution/docker/docker-compose.yml +++ b/distribution/docker/docker-compose.yml @@ -51,7 +51,7 @@ services: - ZOO_MY_ID=1 coordinator: - image: apache/druid:28.0.0 + image: apache/druid:29.0.0 container_name: coordinator volumes: - druid_shared:/opt/shared @@ -67,7 +67,7 @@ services: - environment broker: - image: apache/druid:28.0.0 + image: apache/druid:29.0.0 container_name: broker volumes: - broker_var:/opt/druid/var @@ -83,7 +83,7 @@ services: - environment historical: - image: apache/druid:28.0.0 + image: apache/druid:29.0.0 container_name: historical volumes: - druid_shared:/opt/shared @@ -100,7 +100,7 @@ services: - environment middlemanager: - image: apache/druid:28.0.0 + image: apache/druid:29.0.0 container_name: middlemanager volumes: - druid_shared:/opt/shared @@ -118,7 +118,7 @@ services: - environment router: - image: apache/druid:28.0.0 + image: apache/druid:29.0.0 container_name: router volumes: - router_var:/opt/druid/var diff --git a/distribution/pom.xml b/distribution/pom.xml index ecc00a9155d2..ab8e082c9ab8 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/extensions-contrib/aliyun-oss-extensions/pom.xml b/extensions-contrib/aliyun-oss-extensions/pom.xml index 43f7d558adb7..a33a952c286c 100644 --- a/extensions-contrib/aliyun-oss-extensions/pom.xml +++ b/extensions-contrib/aliyun-oss-extensions/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 61400d13487b..eabf46bfcda9 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 458bb761376c..9dc2faefcec4 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index de4466cca361..e146dcb86ed6 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/compressed-bigdecimal/pom.xml b/extensions-contrib/compressed-bigdecimal/pom.xml index 5a7b278f8b3a..0b30e423bc3f 100644 --- a/extensions-contrib/compressed-bigdecimal/pom.xml +++ b/extensions-contrib/compressed-bigdecimal/pom.xml @@ -25,7 +25,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index a0c1fb46cbb6..772e5c4d3aea 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml index e7f48e89b10b..b28e659ead81 100644 --- a/extensions-contrib/dropwizard-emitter/pom.xml +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -24,7 +24,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-iceberg-extensions/pom.xml b/extensions-contrib/druid-iceberg-extensions/pom.xml index 2765d2aa5935..3e821c01a134 100644 --- a/extensions-contrib/druid-iceberg-extensions/pom.xml +++ b/extensions-contrib/druid-iceberg-extensions/pom.xml @@ -29,7 +29,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/gce-extensions/pom.xml b/extensions-contrib/gce-extensions/pom.xml index 626b9e27c4b1..675dbcb15047 100644 --- a/extensions-contrib/gce-extensions/pom.xml +++ b/extensions-contrib/gce-extensions/pom.xml @@ -21,7 +21,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 1a2767a0330c..205a91994f76 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/influx-extensions/pom.xml b/extensions-contrib/influx-extensions/pom.xml index 966d93c4825c..4ef7f51d5c36 100644 --- a/extensions-contrib/influx-extensions/pom.xml +++ b/extensions-contrib/influx-extensions/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/influxdb-emitter/pom.xml b/extensions-contrib/influxdb-emitter/pom.xml index 83cea085ace3..9a0b8c3bc370 100644 --- a/extensions-contrib/influxdb-emitter/pom.xml +++ b/extensions-contrib/influxdb-emitter/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 46ca7e6c49e1..07e0d7f599c4 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kubernetes-overlord-extensions/pom.xml b/extensions-contrib/kubernetes-overlord-extensions/pom.xml index f231521dfa71..346cc3a00a9c 100644 --- a/extensions-contrib/kubernetes-overlord-extensions/pom.xml +++ b/extensions-contrib/kubernetes-overlord-extensions/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/materialized-view-maintenance/pom.xml b/extensions-contrib/materialized-view-maintenance/pom.xml index d1e6bfa60234..ff35634c2a70 100644 --- a/extensions-contrib/materialized-view-maintenance/pom.xml +++ b/extensions-contrib/materialized-view-maintenance/pom.xml @@ -22,7 +22,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/materialized-view-selection/pom.xml b/extensions-contrib/materialized-view-selection/pom.xml index e965b7d77593..a21eab306a37 100644 --- a/extensions-contrib/materialized-view-selection/pom.xml +++ b/extensions-contrib/materialized-view-selection/pom.xml @@ -22,7 +22,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/momentsketch/pom.xml b/extensions-contrib/momentsketch/pom.xml index b00dac4ef4b1..8c8dd1a47938 100644 --- a/extensions-contrib/momentsketch/pom.xml +++ b/extensions-contrib/momentsketch/pom.xml @@ -22,7 +22,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/moving-average-query/pom.xml b/extensions-contrib/moving-average-query/pom.xml index 845682b5048b..4ea9f2440467 100644 --- a/extensions-contrib/moving-average-query/pom.xml +++ b/extensions-contrib/moving-average-query/pom.xml @@ -24,7 +24,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/opentelemetry-emitter/pom.xml b/extensions-contrib/opentelemetry-emitter/pom.xml index 58415e7ea3ef..6a79a156579c 100644 --- a/extensions-contrib/opentelemetry-emitter/pom.xml +++ b/extensions-contrib/opentelemetry-emitter/pom.xml @@ -23,7 +23,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/opentsdb-emitter/pom.xml b/extensions-contrib/opentsdb-emitter/pom.xml index d1260b54538b..40e8fecca90b 100644 --- a/extensions-contrib/opentsdb-emitter/pom.xml +++ b/extensions-contrib/opentsdb-emitter/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/prometheus-emitter/pom.xml b/extensions-contrib/prometheus-emitter/pom.xml index 2c0e7bcd6b58..cdfff7984db5 100644 --- a/extensions-contrib/prometheus-emitter/pom.xml +++ b/extensions-contrib/prometheus-emitter/pom.xml @@ -23,7 +23,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index 5c3d6df507b9..0cd80a1bc28e 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index a311abebf36b..7b035c7268eb 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 24c493b5b5eb..f61d73f6b33c 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/tdigestsketch/pom.xml b/extensions-contrib/tdigestsketch/pom.xml index ff17035e7b4b..37525615ad46 100644 --- a/extensions-contrib/tdigestsketch/pom.xml +++ b/extensions-contrib/tdigestsketch/pom.xml @@ -22,7 +22,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 4908107b3656..9864c1bc97f3 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -28,7 +28,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 3d20d83f24b6..886b4bc7dd30 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 64f359751cd6..13181ab91a63 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index eeb40f0f47d6..59174336badf 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/azure-extensions/pom.xml b/extensions-core/azure-extensions/pom.xml index 362c9f43b6af..0d49cd1867cb 100644 --- a/extensions-core/azure-extensions/pom.xml +++ b/extensions-core/azure-extensions/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 16d648f5ff33..c99cd793e8fd 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml index 5f06a2ff8d97..fde6cc3e51d9 100644 --- a/extensions-core/druid-aws-rds-extensions/pom.xml +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 423ae175fc8c..707bf51bde20 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-bloom-filter/pom.xml b/extensions-core/druid-bloom-filter/pom.xml index 3377b6910d57..8ad65ff617f8 100644 --- a/extensions-core/druid-bloom-filter/pom.xml +++ b/extensions-core/druid-bloom-filter/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-catalog/pom.xml b/extensions-core/druid-catalog/pom.xml index eb3fdde41d30..a2eb434c1a21 100644 --- a/extensions-core/druid-catalog/pom.xml +++ b/extensions-core/druid-catalog/pom.xml @@ -31,7 +31,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index 1cef05e90e18..d998b2715fa2 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-pac4j/pom.xml b/extensions-core/druid-pac4j/pom.xml index a8cb8b3a08bf..7e629f05549a 100644 --- a/extensions-core/druid-pac4j/pom.xml +++ b/extensions-core/druid-pac4j/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-ranger-security/pom.xml b/extensions-core/druid-ranger-security/pom.xml index 0b120acb7512..d30f1a2d2360 100644 --- a/extensions-core/druid-ranger-security/pom.xml +++ b/extensions-core/druid-ranger-security/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/ec2-extensions/pom.xml b/extensions-core/ec2-extensions/pom.xml index 415fa6c5a960..37c4a3a910f4 100644 --- a/extensions-core/ec2-extensions/pom.xml +++ b/extensions-core/ec2-extensions/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/google-extensions/pom.xml b/extensions-core/google-extensions/pom.xml index 26e036921282..5e35122a6708 100644 --- a/extensions-core/google-extensions/pom.xml +++ b/extensions-core/google-extensions/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 5e0799f118db..2be63de0286f 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 2f9ef866f920..96176516e13e 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 0656bd6c14c0..7e14613a95ee 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 2fe0e4c7e538..d2a3fe76971f 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kinesis-indexing-service/pom.xml b/extensions-core/kinesis-indexing-service/pom.xml index 54ed063b96b2..4ac0d4ec3be1 100644 --- a/extensions-core/kinesis-indexing-service/pom.xml +++ b/extensions-core/kinesis-indexing-service/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kubernetes-extensions/pom.xml b/extensions-core/kubernetes-extensions/pom.xml index 0f62ce90c42e..895fb9f219e5 100644 --- a/extensions-core/kubernetes-extensions/pom.xml +++ b/extensions-core/kubernetes-extensions/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 3aca23fc9ec6..7ed1de772fd5 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 3f4c87c97c9f..a73339d11c8b 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/multi-stage-query/pom.xml b/extensions-core/multi-stage-query/pom.xml index c00676597ff9..59184f2441ba 100644 --- a/extensions-core/multi-stage-query/pom.xml +++ b/extensions-core/multi-stage-query/pom.xml @@ -31,7 +31,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 5e3ca6b1125b..a390fb348887 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/orc-extensions/pom.xml b/extensions-core/orc-extensions/pom.xml index b92a0123e5b3..1c48bf2c17ee 100644 --- a/extensions-core/orc-extensions/pom.xml +++ b/extensions-core/orc-extensions/pom.xml @@ -26,7 +26,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/parquet-extensions/pom.xml b/extensions-core/parquet-extensions/pom.xml index d5b882d60e49..4951c1c49e8d 100644 --- a/extensions-core/parquet-extensions/pom.xml +++ b/extensions-core/parquet-extensions/pom.xml @@ -27,7 +27,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 6ba4bcdb10f8..0febafe9abe3 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index cc39430b96ff..089f6c514ec1 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 579d3ae14a9e..ee99538442b6 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 30300d55b064..fc617482a14f 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -22,7 +22,7 @@ druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index c633dd14142c..c2beb98db97f 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/testing-tools/pom.xml b/extensions-core/testing-tools/pom.xml index 66e8ef707dad..e08de12d8873 100644 --- a/extensions-core/testing-tools/pom.xml +++ b/extensions-core/testing-tools/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 9481b78cc51d..5995e9bb2308 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 7aacd1b5f43b..6950fd2cf815 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 0ed0da875c50..a2b5e596671a 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -31,7 +31,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index 25c6c2823cdd..d388ef2676cb 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -46,7 +46,7 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven druid org.apache.druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/integration-tests-ex/tools/pom.xml b/integration-tests-ex/tools/pom.xml index c0c675f56071..5096abd409e1 100644 --- a/integration-tests-ex/tools/pom.xml +++ b/integration-tests-ex/tools/pom.xml @@ -31,7 +31,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT ../../pom.xml diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 738ac391da52..492a6bda330b 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 6be27033ff92..bbfd3517c4c4 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT pom Druid diff --git a/processing/pom.xml b/processing/pom.xml index 3401a248f509..7f4a6d7fffd8 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 6154ecfa4c20..c2b8269cf2a3 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index e136ea25cb35..c94fb0714cd7 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/sql/pom.xml b/sql/pom.xml index e8be934e512c..d6de3ff0bd93 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -29,7 +29,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/web-console/package-lock.json b/web-console/package-lock.json index 3e396e22fcdc..3aad3ec2b956 100644 --- a/web-console/package-lock.json +++ b/web-console/package-lock.json @@ -1,12 +1,12 @@ { "name": "web-console", - "version": "28.0.0", + "version": "29.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "web-console", - "version": "28.0.0", + "version": "29.0.0", "license": "Apache-2.0", "dependencies": { "@blueprintjs/core": "^4.20.1", diff --git a/web-console/package.json b/web-console/package.json index f8f0310bb530..d43d6615ba41 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -1,6 +1,6 @@ { "name": "web-console", - "version": "28.0.0", + "version": "29.0.0", "description": "A web console for Apache Druid", "author": "Apache Druid Developers ", "license": "Apache-2.0", diff --git a/web-console/pom.xml b/web-console/pom.xml index ec230b468ad7..04f684621343 100644 --- a/web-console/pom.xml +++ b/web-console/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 28.0.0-SNAPSHOT + 29.0.0-SNAPSHOT diff --git a/web-console/unified-console.html b/web-console/unified-console.html index c1edac496fd8..43ef39c6c5f0 100644 --- a/web-console/unified-console.html +++ b/web-console/unified-console.html @@ -71,6 +71,6 @@ }; - + From dba0246aca7cabbff3a5f8819467847834040077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien?= Date: Wed, 11 Oct 2023 06:48:42 -0400 Subject: [PATCH 3/6] Added UI support for waitTillSegmentsLoad (#15110) This relies on the work done in #14322 and #15076. It allows the user to set waitTillSegmentsLoad in the query context (if they want, else it defaults to true) and shows the results in the UI : --- .../druid-models/execution/execution.spec.ts | 3 ++ .../src/druid-models/execution/execution.ts | 50 +++++++++++++++++++ .../query-context/query-context.tsx | 16 ++++++ .../workbench-query/workbench-query.spec.ts | 1 + .../workbench-query/workbench-query.ts | 1 + .../helpers/execution/sql-task-execution.ts | 13 ++++- .../execution-details-pane.spec.tsx.snap | 5 ++ .../execution-details-pane.tsx | 30 ++++++++++- .../execution-progress-bar-pane.spec.tsx.snap | 3 ++ .../execution-progress-bar-pane.tsx | 4 ++ .../ingest-success-pane.spec.tsx.snap | 1 + .../ingest-success-pane.tsx | 6 ++- .../workbench-view/run-panel/run-panel.tsx | 12 +++++ 13 files changed, 141 insertions(+), 4 deletions(-) diff --git a/web-console/src/druid-models/execution/execution.spec.ts b/web-console/src/druid-models/execution/execution.spec.ts index a6b55db68085..fac129baf9b6 100644 --- a/web-console/src/druid-models/execution/execution.spec.ts +++ b/web-console/src/druid-models/execution/execution.spec.ts @@ -268,6 +268,7 @@ describe('Execution', () => { "maxNumTasks": 2, }, "result": undefined, + "segmentStatus": undefined, "sqlQuery": "REPLACE INTO \\"kttm_simple\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", @@ -643,6 +644,7 @@ describe('Execution', () => { "sqlQuery": undefined, "sqlQueryId": undefined, }, + "segmentStatus": undefined, "sqlQuery": undefined, "stages": undefined, "startTime": 2023-07-05T21:33:19.147Z, @@ -679,6 +681,7 @@ describe('Execution', () => { "nativeQuery": undefined, "queryContext": undefined, "result": undefined, + "segmentStatus": undefined, "sqlQuery": undefined, "stages": undefined, "startTime": 2023-07-05T21:40:39.986Z, diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index 8f30711b2838..1899f1260724 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -164,6 +164,18 @@ function formatPendingMessage( } } +interface SegmentStatus { + duration: number; + onDemandSegments: number; + pendingSegments: number; + precachedSegments: number; + startTime: Date; + state: 'INIT' | 'WAITING' | 'SUCCESS'; + totalSegments: number; + unknownSegments: number; + usedSegments: number; +} + export interface ExecutionValue { engine: DruidEngine; id: string; @@ -182,6 +194,7 @@ export interface ExecutionValue { warnings?: ExecutionError[]; capacityInfo?: CapacityInfo; _payload?: MsqTaskPayloadResponse; + segmentStatus?: SegmentStatus; } export class Execution { @@ -292,6 +305,11 @@ export class Execution { const startTime = new Date(deepGet(taskReport, 'multiStageQuery.payload.status.startTime')); const durationMs = deepGet(taskReport, 'multiStageQuery.payload.status.durationMs'); + const segmentLoaderStatus = deepGet( + taskReport, + 'multiStageQuery.payload.status.segmentLoadWaiterStatus', + ); + let result: QueryResult | undefined; const resultsPayload: { signature: { name: string; type: string }[]; @@ -313,6 +331,7 @@ export class Execution { engine: 'sql-msq-task', id, status: Execution.normalizeTaskStatus(status), + segmentStatus: segmentLoaderStatus, startTime: isNaN(startTime.getTime()) ? undefined : startTime, duration: typeof durationMs === 'number' ? durationMs : undefined, usageInfo: getUsageInfoFromStatusPayload( @@ -369,6 +388,7 @@ export class Execution { public readonly error?: ExecutionError; public readonly warnings?: ExecutionError[]; public readonly capacityInfo?: CapacityInfo; + public readonly segmentStatus?: SegmentStatus; public readonly _payload?: { payload: any; task: string }; @@ -390,6 +410,7 @@ export class Execution { this.error = value.error; this.warnings = nonEmptyArray(value.warnings) ? value.warnings : undefined; this.capacityInfo = value.capacityInfo; + this.segmentStatus = value.segmentStatus; this._payload = value._payload; } @@ -412,6 +433,7 @@ export class Execution { error: this.error, warnings: this.warnings, capacityInfo: this.capacityInfo, + segmentStatus: this.segmentStatus, _payload: this._payload, }; @@ -526,6 +548,34 @@ export class Execution { return status !== 'SUCCESS' && status !== 'FAILED'; } + public getSegmentStatusDescription() { + const { segmentStatus } = this; + + let label = ''; + + switch (segmentStatus?.state) { + case 'INIT': + label = 'Waiting for segments loading to start...'; + break; + + case 'WAITING': + label = 'Waiting for segments loading to complete...'; + break; + + case 'SUCCESS': + label = 'Segments loaded successfully in ' + segmentStatus.duration + 'ms.'; + break; + + default: + break; + } + + return { + label, + ...segmentStatus, + }; + } + public isFullyComplete(): boolean { if (this.isWaitingForQuery()) return false; diff --git a/web-console/src/druid-models/query-context/query-context.tsx b/web-console/src/druid-models/query-context/query-context.tsx index d09fa73868ee..cdaf8dd84e35 100644 --- a/web-console/src/druid-models/query-context/query-context.tsx +++ b/web-console/src/druid-models/query-context/query-context.tsx @@ -162,6 +162,22 @@ export function changeFinalizeAggregations( : deepDelete(context, 'finalizeAggregations'); } +// waitTillSegmentsLoad + +export function getWaitTillSegmentsLoad(context: QueryContext): boolean | undefined { + const { waitTillSegmentsLoad } = context; + return typeof waitTillSegmentsLoad === 'boolean' ? waitTillSegmentsLoad : undefined; +} + +export function changeWaitTillSegmentsLoad( + context: QueryContext, + waitTillSegmentsLoad: boolean | undefined, +): QueryContext { + return typeof waitTillSegmentsLoad === 'boolean' + ? deepSet(context, 'waitTillSegmentsLoad', waitTillSegmentsLoad) + : deepDelete(context, 'waitTillSegmentsLoad'); +} + // groupByEnableMultiValueUnnesting export function getGroupByEnableMultiValueUnnesting(context: QueryContext): boolean | undefined { diff --git a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts index 8732b93d4248..079ccde5a2dd 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts @@ -423,6 +423,7 @@ describe('WorkbenchQuery', () => { finalizeAggregations: false, groupByEnableMultiValueUnnesting: false, useCache: false, + waitTillSegmentsLoad: true, }, header: true, query: 'INSERT INTO wiki2 SELECT * FROM wikipedia', diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index d7847287fa64..43fe3ea6528c 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -552,6 +552,7 @@ export class WorkbenchQuery { apiQuery.context.executionMode ??= 'async'; apiQuery.context.finalizeAggregations ??= !ingestQuery; apiQuery.context.groupByEnableMultiValueUnnesting ??= !ingestQuery; + apiQuery.context.waitTillSegmentsLoad ??= true; } if (Array.isArray(queryParameters) && queryParameters.length) { diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts index d41370df16cf..75b82d17b868 100644 --- a/web-console/src/helpers/execution/sql-task-execution.ts +++ b/web-console/src/helpers/execution/sql-task-execution.ts @@ -57,7 +57,13 @@ export interface SubmitTaskQueryOptions { export async function submitTaskQuery( options: SubmitTaskQueryOptions, ): Promise> { - const { query, context, prefixLines, cancelToken, preserveOnTermination, onSubmitted } = options; + const { query, prefixLines, cancelToken, preserveOnTermination, onSubmitted } = options; + + // setting waitTillSegmentsLoad to true by default + const context = { + waitTillSegmentsLoad: true, + ...(options.context || {}), + }; let sqlQuery: string; let jsonQuery: Record; @@ -261,6 +267,11 @@ export async function updateExecutionWithDatasourceLoadedIfNeeded( return execution; } + // This means we don't have to perform the SQL query to check if the segments are loaded + if (execution.queryContext?.waitTillSegmentsLoad === true) { + return execution.markDestinationDatasourceLoaded(); + } + const endTime = execution.getEndTime(); if ( !endTime || // If endTime is not set (this is not expected to happen) then just bow out diff --git a/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap index 6f3c23bee00d..df9acdfb2cc1 100644 --- a/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-details-pane/__snapshots__/execution-details-pane.spec.tsx.snap @@ -22,6 +22,7 @@ exports[`ExecutionDetailsPane matches snapshot no init tab 1`] = ` "id": "native", "label": "Native query", }, + false, undefined, undefined, Object { @@ -286,6 +287,7 @@ PARTITIONED BY DAY", "maxParseExceptions": 2, }, "result": undefined, + "segmentStatus": undefined, "sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", @@ -909,6 +911,7 @@ PARTITIONED BY DAY", "maxParseExceptions": 2, }, "result": undefined, + "segmentStatus": undefined, "sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", @@ -1319,6 +1322,7 @@ exports[`ExecutionDetailsPane matches snapshot with init tab 1`] = ` "id": "native", "label": "Native query", }, + false, undefined, undefined, Object { @@ -1576,6 +1580,7 @@ PARTITIONED BY DAY", "maxParseExceptions": 2, }, "result": undefined, + "segmentStatus": undefined, "sqlQuery": "REPLACE INTO \\"kttm-blank-lines\\" OVERWRITE ALL SELECT TIME_PARSE(\\"timestamp\\") AS \\"__time\\", diff --git a/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx b/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx index 72a6350d7ba1..0f1200a8661f 100644 --- a/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx +++ b/web-console/src/views/workbench-view/execution-details-pane/execution-details-pane.tsx @@ -23,7 +23,7 @@ import React, { useState } from 'react'; import { FancyTabPane } from '../../../components'; import type { Execution } from '../../../druid-models'; -import { pluralIfNeeded } from '../../../utils'; +import { formatDuration, formatDurationWithMs, pluralIfNeeded } from '../../../utils'; import { DestinationPagesPane } from '../destination-pages-pane/destination-pages-pane'; import { ExecutionErrorPane } from '../execution-error-pane/execution-error-pane'; import { ExecutionStagesPane } from '../execution-stages-pane/execution-stages-pane'; @@ -40,7 +40,8 @@ export type ExecutionDetailsTab = | 'result' | 'pages' | 'error' - | 'warnings'; + | 'warnings' + | 'segmentStatus'; interface ExecutionDetailsPaneProps { execution: Execution; @@ -53,6 +54,7 @@ export const ExecutionDetailsPane = React.memo(function ExecutionDetailsPane( ) { const { execution, initTab, goToTask } = props; const [activeTab, setActiveTab] = useState(initTab || 'general'); + const segmentStatusDescription = execution.getSegmentStatusDescription(); function renderContent() { switch (activeTab) { @@ -120,6 +122,25 @@ export const ExecutionDetailsPane = React.memo(function ExecutionDetailsPane( case 'warnings': return ; + case 'segmentStatus': + return ( + <> +

+ Duration:{' '} + {segmentStatusDescription.duration + ? formatDurationWithMs(segmentStatusDescription.duration) + : '-'} + {execution.duration + ? ` (query duration was ${formatDuration(execution.duration)})` + : ''} +

+

Total segments: {segmentStatusDescription.totalSegments ?? '-'}

+

Used segments: {segmentStatusDescription.usedSegments ?? '-'}

+

Precached segments: {segmentStatusDescription.precachedSegments ?? '-'}

+

On demand segments: {segmentStatusDescription.onDemandSegments ?? '-'}

+ + ); + default: return; } @@ -146,6 +167,11 @@ export const ExecutionDetailsPane = React.memo(function ExecutionDetailsPane( label: 'Native query', icon: IconNames.COG, }, + Boolean(execution.segmentStatus) && { + id: 'segmentStatus', + label: 'Segments', + icon: IconNames.HEAT_GRID, + }, execution.result && { id: 'result', label: 'Results', diff --git a/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap index 68a8e3103c98..3e2b45ec2f1a 100644 --- a/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap +++ b/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap @@ -20,5 +20,8 @@ exports[`ExecutionProgressBarPane matches snapshot 1`] = ` className="overall" intent="primary" /> + + + `; diff --git a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx index 349bdaff93dd..4520c20a39b1 100644 --- a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx +++ b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx @@ -50,6 +50,9 @@ export const ExecutionProgressBarPane = React.memo(function ExecutionProgressBar const idx = stages ? stages.currentStageIndex() : -1; const waitingForSegments = stages && !execution.isWaitingForQuery(); + + const segmentStatusDescription = execution?.getSegmentStatusDescription(); + return (