From 294556da1c649b7b652dddaf20841774142992f8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 27 Aug 2023 22:12:53 +0530 Subject: [PATCH 01/82] move smc to coordinator --- ...ruidSchemaInternRowSignatureBenchmark.java | 10 +- .../timeline/SegmentStatusInCluster.java | 51 ++- .../timeline/SegmentStatusInClusterTest.java | 2 +- .../apache/druid/client/BrokerServerView.java | 11 + .../druid/client/CoordinatorServerView.java | 250 +++++++++++- ...ryConfig.java => InternalQueryConfig.java} | 2 +- .../client/coordinator/CoordinatorClient.java | 6 + .../coordinator/CoordinatorClientImpl.java | 19 + .../metadata}/AvailableSegmentMetadata.java | 15 +- .../segment/metadata/DatasourceSchema.java | 32 ++ .../metadata}/SegmentMetadataCache.java | 105 +++-- .../metadata/SegmentMetadataCacheConfig.java | 46 +++ .../server/http/DataSourcesResource.java | 4 +- .../druid/server/http/MetadataResource.java | 47 ++- ...Test.java => InternalQueryConfigTest.java} | 20 +- .../server/http/DataSourcesResourceTest.java | 8 +- .../server/http/MetadataResourceTest.java | 8 +- .../java/org/apache/druid/cli/CliBroker.java | 6 +- .../org/apache/druid/cli/CliCoordinator.java | 45 +++ .../calcite/planner/CalcitePlannerModule.java | 3 +- .../planner/SegmentMetadataCacheConfig.java | 100 ----- .../schema/BrokerSegmentMetadataCache.java | 75 ++++ .../BrokerSegmentMetadataCacheConfig.java | 31 ++ .../schema/BrokerSegmentMetadataView.java | 360 ++++++++++++++++++ .../schema/DruidCalciteSchemaModule.java | 23 +- .../druid/sql/calcite/schema/DruidSchema.java | 20 +- .../calcite/schema/MetadataSegmentView.java | 242 ------------ .../PhysicalDatasourceMetadataBuilder.java | 42 ++ .../sql/calcite/schema/SystemSchema.java | 54 ++- .../org/apache/druid/sql/guice/SqlModule.java | 2 +- .../SegmentMetadataCacheConfigTest.java | 3 +- .../schema/DruidSchemaNoDataInitTest.java | 7 +- .../SegmentDataCacheConcurrencyTest.java | 9 +- .../schema/SegmentMetadataCacheCommon.java | 2 +- .../schema/SegmentMetadataCacheTest.java | 38 +- .../sql/calcite/schema/SystemSchemaTest.java | 23 +- .../druid/sql/calcite/util/CalciteTests.java | 6 +- .../sql/calcite/util/QueryFrameworkUtils.java | 11 +- 38 files changed, 1225 insertions(+), 513 deletions(-) rename server/src/main/java/org/apache/druid/client/{BrokerInternalQueryConfig.java => InternalQueryConfig.java} (97%) rename {sql/src/main/java/org/apache/druid/sql/calcite/schema => server/src/main/java/org/apache/druid/segment/metadata}/AvailableSegmentMetadata.java (90%) create mode 100644 server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java rename {sql/src/main/java/org/apache/druid/sql/calcite/schema => server/src/main/java/org/apache/druid/segment/metadata}/SegmentMetadataCache.java (92%) create mode 100644 server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java rename server/src/test/java/org/apache/druid/client/{BrokerInternalQueryConfigTest.java => InternalQueryConfigTest.java} (84%) delete mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfig.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java delete mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 138509a1a80e..e87efb348079 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequence; @@ -38,8 +38,8 @@ import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.planner.PlannerConfig; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.schema.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.LinearShardSpec; @@ -80,7 +80,7 @@ public SegmentMetadataCacheForBenchmark( final JoinableFactory joinableFactory, final PlannerConfig config, final Escalator escalator, - final BrokerInternalQueryConfig brokerInternalQueryConfig + final InternalQueryConfig internalQueryConfig ) { super( @@ -90,7 +90,7 @@ public SegmentMetadataCacheForBenchmark( joinableFactory, SegmentMetadataCacheConfig.create(), escalator, - brokerInternalQueryConfig, + internalQueryConfig, new NoopServiceEmitter() ); } diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index 4e5577f76039..26d89d79ed34 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -53,25 +53,43 @@ public class SegmentStatusInCluster implements Comparable callback.segmentAdded(server, segment)); + } + + private void serverAddedSegment2(final DruidServerMetadata server, final DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + synchronized (lock) { + // in theory we could probably just filter this to ensure we don't put ourselves in here, to make broker tree + // query topologies, but for now just skip all brokers, so we don't create some sort of wild infinite query + // loop... + if (!server.getType().equals(ServerType.BROKER)) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + ServerSelector selector = selectors.get(segmentId); + if (selector == null) { + selector = new ServerSelector(segment, tierSelectorStrategy); + + VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); + if (timeline == null) { + // broker needs to skip tombstones + timeline = new VersionedIntervalTimeline<>(Ordering.natural(), true); + timelines2.put(segment.getDataSource(), timeline); + } + + timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); + selectors.put(segmentId, selector); + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); + if (inventoryValue == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping addition of segment[%s].", + server.getName(), + segmentId + ); + return; + } else { + queryableDruidServer = addServer(inventoryValue); + } + } + selector.addServerAndUpdateSegment(queryableDruidServer, segment); + } + // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it + runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); + } + } + + private QueryableDruidServer addServer(DruidServer server) + { + QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); + QueryableDruidServer exists = clients.put(server.getName(), retVal); + if (exists != null) { + log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); + } + + return retVal; + } + + private DirectDruidClient makeDirectClient(DruidServer server) + { + return new DirectDruidClient( + warehouse, + queryWatcher, + smileMapper, + httpClient, + server.getScheme(), + server.getHost(), + emitter + ); } private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) @@ -178,7 +296,9 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen log.warn("Told to remove non-existant segment[%s]", segmentId); return; } - segmentLoadInfo.removeServer(server); + if (segmentLoadInfo.removeServer(server)) { + runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } if (segmentLoadInfo.isEmpty()) { VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); segmentLoadInfos.remove(segmentId); @@ -197,6 +317,62 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen segment.getInterval(), segment.getVersion() ); + } else { + runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); + } + } + } + } + + private void serverRemovedSegment2(DruidServerMetadata server, DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + final ServerSelector selector; + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + // we don't store broker segments here, but still run the callbacks for the segment being removed from the server + // since the broker segments are not stored on the timeline, do not fire segmentRemoved event + selector = selectors.get(segmentId); + if (selector == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping removal of segment[%s].", + server.getName(), + segmentId + ); + } else if (!selector.removeServer(queryableDruidServer)) { + log.warn( + "Asked to disassociate non-existant association between server[%s] and segment[%s]", + server, + segmentId + ); + } else { + // runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } + + if (selector.isEmpty()) { + VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); + selectors.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } else { + // runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); } } } @@ -210,9 +386,9 @@ public VersionedIntervalTimeline getTimeline(DataSource } } - public Map getSegmentLoadInfos() + public Set getLoadedSegmentIds() { - return segmentLoadInfos; + return segmentLoadInfos.keySet(); } @Override @@ -238,4 +414,68 @@ public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) { return baseView.isSegmentLoadedByServer(serverKey, segment); } + + private void runTimelineCallbacks(final Function function) + { + for (Map.Entry entry : timelineCallbacks.entrySet()) { + entry.getValue().execute( + () -> { + if (CallbackAction.UNREGISTER == function.apply(entry.getKey())) { + timelineCallbacks.remove(entry.getKey()); + } + } + ); + } + } + + @Override + public void registerTimelineCallback(Executor exec, TimelineCallback callback) + { + timelineCallbacks.put(callback, exec); + } + + @Override + public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) + { + throw new UnsupportedOperationException(); + } + + @Override + public void registerSegmentCallback(Executor exec, SegmentCallback callback) + { + baseView.registerSegmentCallback(exec, callback); + } + + @Override + public Optional> getTimeline(DataSourceAnalysis analysis) + { + final TableDataSource table = + analysis.getBaseTableDataSource() + .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); + + synchronized (lock) { + return Optional.ofNullable(timelines2.get(table.getName())); + } + } + + @Override + public List getDruidServers() + { + return clients.values().stream() + .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) + .collect(Collectors.toList()); + } + + @Override + public QueryRunner getQueryRunner(DruidServer server) + { + synchronized (lock) { + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.error("No QueryRunner found for server name[%s].", server.getName()); + return null; + } + return queryableDruidServer.getQueryRunner(); + } + } } diff --git a/server/src/main/java/org/apache/druid/client/BrokerInternalQueryConfig.java b/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java similarity index 97% rename from server/src/main/java/org/apache/druid/client/BrokerInternalQueryConfig.java rename to server/src/main/java/org/apache/druid/client/InternalQueryConfig.java index 9b893778d60b..3c1d4d96022b 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerInternalQueryConfig.java +++ b/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java @@ -29,7 +29,7 @@ * should add to their query payload. The runtime properties for this class * have the prefix "druid.broker.internal.query.config." */ -public class BrokerInternalQueryConfig +public class InternalQueryConfig { @JsonProperty private Map context = new HashMap<>(); diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index c497dcb68940..812f35d4622e 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -20,12 +20,16 @@ package org.apache.druid.client.coordinator; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentStatusInCluster; import org.joda.time.Interval; +import java.util.Iterator; import java.util.List; +import java.util.Set; public interface CoordinatorClient { @@ -44,6 +48,8 @@ public interface CoordinatorClient */ ListenableFuture> fetchUsedSegments(String dataSource, List intervals); + ListenableFuture> fetchDatasourceSchema(List datasources); + /** * Returns a new instance backed by a ServiceClient which follows the provided retryPolicy */ diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index e023d9827bd4..dae00c73bea0 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -26,15 +26,19 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler; +import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; import org.apache.druid.rpc.ServiceRetryPolicy; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentStatusInCluster; import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; +import java.util.Iterator; import java.util.List; +import java.util.Set; public class CoordinatorClientImpl implements CoordinatorClient { @@ -106,6 +110,21 @@ public ListenableFuture> fetchUsedSegments(String dataSource, ); } + @Override + public ListenableFuture> fetchDatasourceSchema(List datasources) + { + final String path = "/druid/coordinator/v1/metadata/datasourceSchema"; + + return FutureUtils.transform( + client.asyncRequest( + new RequestBuilder(HttpMethod.POST, path) + .jsonContent(jsonMapper, datasources), + new BytesFullResponseHandler() + ), + holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) + ); + } + @Override public CoordinatorClientImpl withRetryPolicy(ServiceRetryPolicy retryPolicy) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java similarity index 90% rename from sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java rename to server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java index 20d9b7e0ad9d..104194531769 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -27,7 +27,7 @@ import java.util.Set; /** - * Immutable representation of RowSignature and other segment attributes needed by {@link SystemSchema.SegmentsTable} + * Immutable representation of RowSignature and other segment attributes needed by {@code SystemSchema.SegmentsTable} * This class contains the metadata of segments announced by historicals or ingestion tasks. */ public class AvailableSegmentMetadata @@ -159,4 +159,15 @@ public AvailableSegmentMetadata build() } } + @Override + public String toString() + { + return "AvailableSegmentMetadata{" + + "segment=" + segment + + ", isRealtime=" + isRealtime + + ", segmentServers=" + segmentServers + + ", numRows=" + numRows + + ", rowSignature=" + rowSignature + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java new file mode 100644 index 000000000000..cad5cb882c62 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java @@ -0,0 +1,32 @@ +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.column.RowSignature; + +public class DatasourceSchema +{ + private final String datasource; + private final RowSignature rowSignature; + + @JsonCreator + public DatasourceSchema( + @JsonProperty("datasource") String datasource, + @JsonProperty("rowSignature") RowSignature rowSignature) + { + this.datasource = datasource; + this.rowSignature = rowSignature; + } + + @JsonProperty + public String getDatasource() + { + return datasource; + } + + @JsonProperty + public RowSignature getRowSignature() + { + return rowSignature; + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java similarity index 92% rename from sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java rename to server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 37ff3bda313b..971c45cd3885 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import com.fasterxml.jackson.annotation.JsonCreator; import com.google.common.annotations.VisibleForTesting; @@ -25,6 +25,7 @@ import com.google.common.base.Predicates; import com.google.common.base.Stopwatch; import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Interner; import com.google.common.collect.Interners; @@ -33,7 +34,7 @@ import com.google.common.collect.Sets; import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.inject.Inject; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.guice.ManageLifecycle; @@ -50,7 +51,6 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; @@ -61,15 +61,11 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.Types; -import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.SegmentManager; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Escalator; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; @@ -118,8 +114,7 @@ public class SegmentMetadataCache private final SegmentMetadataCacheConfig config; // Escalator, so we can attach an authentication result to queries we generate. private final Escalator escalator; - private final SegmentManager segmentManager; - private final JoinableFactory joinableFactory; + private final ExecutorService cacheExec; private final ExecutorService callbackExec; private final ServiceEmitter emitter; @@ -129,7 +124,7 @@ public class SegmentMetadataCache * Map of DataSource -> DruidTable. * This map can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final ConcurrentMap tables = new ConcurrentHashMap<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -186,7 +181,7 @@ public class SegmentMetadataCache * Currently, there are 2 threads that can access these variables. * * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DatasourceTable} if necessary + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DatasourceSchema} if necessary * based on the information collected via timeline callbacks. */ private final Object lock = new Object(); @@ -204,7 +199,7 @@ public class SegmentMetadataCache private final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); // Configured context to attach to internally generated queries. - private final BrokerInternalQueryConfig brokerInternalQueryConfig; + private final InternalQueryConfig internalQueryConfig; @GuardedBy("lock") private boolean refreshImmediately = false; @@ -223,24 +218,20 @@ public class SegmentMetadataCache public SegmentMetadataCache( final QueryLifecycleFactory queryLifecycleFactory, final TimelineServerView serverView, - final SegmentManager segmentManager, - final JoinableFactory joinableFactory, final SegmentMetadataCacheConfig config, final Escalator escalator, - final BrokerInternalQueryConfig brokerInternalQueryConfig, + final InternalQueryConfig internalQueryConfig, final ServiceEmitter emitter ) { this.queryLifecycleFactory = Preconditions.checkNotNull(queryLifecycleFactory, "queryLifecycleFactory"); Preconditions.checkNotNull(serverView, "serverView"); - this.segmentManager = segmentManager; - this.joinableFactory = joinableFactory; this.config = Preconditions.checkNotNull(config, "config"); this.columnTypeMergePolicy = config.getMetadataColumnTypeMergePolicy(); this.cacheExec = Execs.singleThreaded("DruidSchema-Cache-%d"); this.callbackExec = Execs.singleThreaded("DruidSchema-Callback-%d"); this.escalator = escalator; - this.brokerInternalQueryConfig = brokerInternalQueryConfig; + this.internalQueryConfig = internalQueryConfig; this.emitter = emitter; initServerViewTimelineCallback(serverView); @@ -421,19 +412,23 @@ void refresh(final Set segmentsToRefresh, final Set dataSourc // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - final DatasourceTable.PhysicalDatasourceMetadata druidTable = buildDruidTable(dataSource); - if (druidTable == null) { - log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tables.remove(dataSource); - continue; - } - final DatasourceTable.PhysicalDatasourceMetadata oldTable = tables.put(dataSource, druidTable); - final String description = druidTable.dataSource().isGlobal() ? "global dataSource" : "dataSource"; - if (oldTable == null || !oldTable.rowSignature().equals(druidTable.rowSignature())) { - log.info("%s [%s] has new signature: %s.", description, dataSource, druidTable.rowSignature()); - } else { - log.debug("%s [%s] signature is unchanged.", description, dataSource); - } + rebuildDatasource(dataSource); + } + } + + public void rebuildDatasource(String dataSource) + { + final DatasourceSchema druidTable = buildDruidTable(dataSource); + if (druidTable == null) { + log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + tables.remove(dataSource); + return; + } + final DatasourceSchema oldTable = tables.put(dataSource, druidTable); + if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + } else { + log.info("[%s] signature is unchanged.", dataSource); } } @@ -449,12 +444,17 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } - protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) + public DatasourceSchema getDatasource(String name) { return tables.get(name); } - protected Set getDatasourceNames() + public Map getDatasourceSchemaMap() + { + return ImmutableMap.copyOf(tables); + } + + public Set getDatasourceNames() { return tables.keySet(); } @@ -717,7 +717,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin throw new ISE("'segments' must all match 'dataSource'!"); } - log.debug("Refreshing metadata for dataSource[%s].", dataSource); + log.info("Refreshing metadata for dataSource[%s].", dataSource); final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); @@ -743,7 +743,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin log.warn("Got analysis for segment [%s] we didn't ask for, ignoring.", analysis.getId()); } else { final RowSignature rowSignature = analysisToRowSignature(analysis); - log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature); + log.info("Segment[%s] has signature[%s].", segmentId, rowSignature); segmentMetadataInfo.compute( dataSource, (datasourceKey, dataSourceSegments) -> { @@ -795,7 +795,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin emitter.emit(builder.build("metadatacache/refresh/time", refreshDurationMillis)); - log.debug( + log.info( "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", dataSource, refreshDurationMillis, @@ -808,7 +808,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin @VisibleForTesting @Nullable - DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(final String dataSource) + public DatasourceSchema buildDruidTable(final String dataSource) { ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); @@ -836,26 +836,10 @@ DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(final String dataSour final RowSignature.Builder builder = RowSignature.builder(); columnTypes.forEach(builder::add); - final TableDataSource tableDataSource; - - // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing - // in the segment cache, which in this case belongs to the broker meaning only broadcast segments live here) - // to be joinable, it must be possibly joinable according to the factory. we only consider broadcast datasources - // at this time, and isGlobal is currently strongly coupled with joinable, so only make a global table datasource - // if also joinable - final GlobalTableDataSource maybeGlobal = new GlobalTableDataSource(dataSource); - final boolean isJoinable = joinableFactory.isDirectlyJoinable(maybeGlobal); - final boolean isBroadcast = segmentManager.getDataSourceNames().contains(dataSource); - if (isBroadcast && isJoinable) { - tableDataSource = maybeGlobal; - } else { - tableDataSource = new TableDataSource(dataSource); - } - return new DatasourceTable.PhysicalDatasourceMetadata(tableDataSource, builder.build(), isJoinable, isBroadcast); + return new DatasourceSchema(dataSource, builder.build()); } - @VisibleForTesting - Map getSegmentMetadataSnapshot() + public Map getSegmentMetadataSnapshot() { final Map segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments); for (ConcurrentSkipListMap val : segmentMetadataInfo.values()) { @@ -864,6 +848,15 @@ Map getSegmentMetadataSnapshot() return segmentMetadata; } + @Nullable + public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId) + { + if (!segmentMetadataInfo.containsKey(datasource)) { + return null; + } + return segmentMetadataInfo.get(datasource).get(segmentId); + } + /** * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. * As a result, the returned value might be inexact. @@ -926,7 +919,7 @@ protected Sequence runSegmentMetadataQuery( false, // disable the parallel merge because we don't care about the merge and don't want to consume its resources QueryContexts.override( - brokerInternalQueryConfig.getContext(), + internalQueryConfig.getContext(), QueryContexts.BROKER_PARALLEL_MERGE_KEY, false ), diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java new file mode 100644 index 000000000000..c59ee23a77b2 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -0,0 +1,46 @@ +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +public class SegmentMetadataCacheConfig +{ + @JsonProperty + private boolean awaitInitializationOnStart = true; + + @JsonProperty + private Period metadataRefreshPeriod = new Period("PT1M"); + + @JsonProperty + private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = + new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); + + public static SegmentMetadataCacheConfig create() + { + return new SegmentMetadataCacheConfig(); + } + + public static SegmentMetadataCacheConfig create( + String metadataRefreshPeriod + ) + { + SegmentMetadataCacheConfig config = new SegmentMetadataCacheConfig(); + config.metadataRefreshPeriod = new Period(metadataRefreshPeriod); + return config; + } + + public boolean isAwaitInitializationOnStart() + { + return awaitInitializationOnStart; + } + + public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + { + return metadataColumnTypeMergePolicy; + } + + public Period getMetadataRefreshPeriod() + { + return metadataRefreshPeriod; + } +} diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 1e146736da13..9fa73970100f 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -487,13 +487,13 @@ public Response getDatasourceLoadstatus( private SegmentsLoadStatistics computeSegmentLoadStatistics(Iterable segments) { - Map segmentLoadInfos = serverInventoryView.getSegmentLoadInfos(); + Set loadedSegmentIds = serverInventoryView.getLoadedSegmentIds(); int numPublishedSegments = 0; int numUnavailableSegments = 0; int numLoadedSegments = 0; for (DataSegment segment : segments) { numPublishedSegments++; - if (!segmentLoadInfos.containsKey(segment.getId())) { + if (!loadedSegmentIds.contains(segment.getId())) { numUnavailableSegments++; } else { numLoadedSegments++; diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 4f9631c1cfd2..9c9622e05e50 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -28,6 +28,9 @@ import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.Segments; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.server.JettyUtils; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -54,7 +57,9 @@ import javax.ws.rs.core.UriInfo; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -69,19 +74,22 @@ public class MetadataResource private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; + private final SegmentMetadataCache segmentMetadataCache; @Inject public MetadataResource( SegmentsMetadataManager segmentsMetadataManager, IndexerMetadataStorageCoordinator metadataStorageCoordinator, AuthorizerMapper authorizerMapper, - DruidCoordinator coordinator + DruidCoordinator coordinator, + SegmentMetadataCache segmentMetadataCache ) { this.segmentsMetadataManager = segmentsMetadataManager; this.metadataStorageCoordinator = metadataStorageCoordinator; this.authorizerMapper = authorizerMapper; this.coordinator = coordinator; + this.segmentMetadataCache = segmentMetadataCache; } @GET @@ -185,21 +193,40 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .flatMap(t -> t.getSegments().stream()); final Set overshadowedSegments = dataSourcesSnapshot.getOvershadowedSegments(); + Set segmentAlreadySeen = new HashSet<>(); final Stream segmentStatus = usedSegments.map(segment -> { // The replication factor for unloaded segments is 0 as they will be unloaded soon boolean isOvershadowed = overshadowedSegments.contains(segment); + AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata(segment.getDataSource(), segment.getId()); Integer replicationFactor = isOvershadowed ? (Integer) 0 : coordinator.getReplicationFactor(segment.getId()); - - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor); + Long numRows = (null != availableSegmentMetadata) ? availableSegmentMetadata.getNumRows() : null; + segmentAlreadySeen.add(segment.getId()); + return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, numRows, 0L, true); }); + final Stream realtimeSegmentStatus = segmentMetadataCache + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment().getId())) + .map(availableSegmentMetadata -> { + return new SegmentStatusInCluster( + availableSegmentMetadata.getSegment(), + false, + (int) availableSegmentMetadata.getNumReplicas(), + availableSegmentMetadata.getNumRows(), + availableSegmentMetadata.isRealtime(), + false + ); + }); + final Function> raGenerator = segment -> Collections .singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSegment().getDataSource())); final Iterable authorizedSegments = AuthorizationUtils.filterAuthorizedResources( req, - segmentStatus::iterator, + Stream.concat(segmentStatus, realtimeSegmentStatus)::iterator, raGenerator, authorizerMapper ); @@ -302,4 +329,16 @@ public Response getUsedSegment( } return Response.status(Response.Status.NOT_FOUND).build(); } + + @POST + @Path("/datasourceSchema") + public Response getDatasourceSchema( + List datasources + ) + { + Map datasourceSchemaMap = segmentMetadataCache.getDatasourceSchemaMap(); + datasourceSchemaMap.keySet().retainAll(datasources); + + return Response.status(Response.Status.OK).entity(datasourceSchemaMap.values()).build(); + } } diff --git a/server/src/test/java/org/apache/druid/client/BrokerInternalQueryConfigTest.java b/server/src/test/java/org/apache/druid/client/InternalQueryConfigTest.java similarity index 84% rename from server/src/test/java/org/apache/druid/client/BrokerInternalQueryConfigTest.java rename to server/src/test/java/org/apache/druid/client/InternalQueryConfigTest.java index 24b61bb1ac4b..1030947e78dd 100644 --- a/server/src/test/java/org/apache/druid/client/BrokerInternalQueryConfigTest.java +++ b/server/src/test/java/org/apache/druid/client/InternalQueryConfigTest.java @@ -39,7 +39,7 @@ import java.util.HashMap; import java.util.Map; -public class BrokerInternalQueryConfigTest +public class InternalQueryConfigTest { private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); @@ -49,11 +49,11 @@ public void testSerde() throws Exception //defaults String json = "{}"; - BrokerInternalQueryConfig config = MAPPER.readValue( + InternalQueryConfig config = MAPPER.readValue( MAPPER.writeValueAsString( - MAPPER.readValue(json, BrokerInternalQueryConfig.class) + MAPPER.readValue(json, InternalQueryConfig.class) ), - BrokerInternalQueryConfig.class + InternalQueryConfig.class ); Assert.assertEquals(ImmutableMap.of(), config.getContext()); @@ -63,9 +63,9 @@ public void testSerde() throws Exception config = MAPPER.readValue( MAPPER.writeValueAsString( - MAPPER.readValue(json, BrokerInternalQueryConfig.class) + MAPPER.readValue(json, InternalQueryConfig.class) ), - BrokerInternalQueryConfig.class + InternalQueryConfig.class ); Map expected = new HashMap<>(); @@ -84,9 +84,9 @@ public void testMalfomattedContext() throws Exception String malformedJson = "{\"priority: 5}"; MAPPER.readValue( MAPPER.writeValueAsString( - MAPPER.readValue(malformedJson, BrokerInternalQueryConfig.class) + MAPPER.readValue(malformedJson, InternalQueryConfig.class) ), - BrokerInternalQueryConfig.class + InternalQueryConfig.class ); } @@ -104,7 +104,7 @@ public void configure(Binder binder) { binder.install(new ConfigModule()); binder.install(new DruidGuiceExtensions()); - JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", BrokerInternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", InternalQueryConfig.class); } @Provides @@ -115,7 +115,7 @@ public ObjectMapper jsonMapper() } } ); - BrokerInternalQueryConfig config = injector.getInstance(BrokerInternalQueryConfig.class); + InternalQueryConfig config = injector.getInstance(InternalQueryConfig.class); Assert.assertEquals(ImmutableMap.of(), config.getContext()); } } diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 551151458546..94b9bf8a84d0 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -1302,7 +1302,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1318,7 +1318,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1381,7 +1381,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1397,7 +1397,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index d5ab6aaaa1a5..717db5f03caa 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -112,11 +112,11 @@ public void testGetAllSegmentsWithOvershadowedStatus() final List resultList = extractSegmentStatusList(response); Assert.assertEquals(resultList.size(), 4); - Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null), resultList.get(1)); - Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1), resultList.get(2)); + Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 5L, 0L, true), resultList.get(0)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 5L, 0L, true), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, 5L, 0L, true), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, 5L, 0L, true), resultList.get(3)); } @Test diff --git a/services/src/main/java/org/apache/druid/cli/CliBroker.java b/services/src/main/java/org/apache/druid/cli/CliBroker.java index dd9c3c3c0b59..849444c8ec83 100644 --- a/services/src/main/java/org/apache/druid/cli/CliBroker.java +++ b/services/src/main/java/org/apache/druid/cli/CliBroker.java @@ -26,7 +26,7 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.name.Names; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.CachingClusteredClient; @@ -70,6 +70,7 @@ import org.apache.druid.server.initialization.jetty.JettyServerInitializer; import org.apache.druid.server.metrics.QueryCountStatsProvider; import org.apache.druid.server.router.TieredBrokerConfig; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.PruneLoadSpec; import org.eclipse.jetty.server.Server; @@ -127,6 +128,7 @@ protected List getModules() binder.bind(CachingClusteredClient.class).in(LazySingleton.class); LifecycleModule.register(binder, BrokerServerView.class); + LifecycleModule.register(binder, BrokerSegmentMetadataView.class); binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); JsonConfigProvider.bind(binder, "druid.broker.cache", CacheConfig.class); @@ -137,7 +139,7 @@ protected List getModules() JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); JsonConfigProvider.bind(binder, "druid.broker.segment", BrokerSegmentWatcherConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", BrokerInternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", InternalQueryConfig.class); binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 5327f0b3a764..93e7dad644e4 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -37,19 +37,33 @@ import com.google.inject.util.Providers; import org.apache.curator.framework.CuratorFramework; import org.apache.druid.audit.AuditManager; +import org.apache.druid.client.BrokerSegmentWatcherConfig; +import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.HttpServerInventoryViewResource; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; +import org.apache.druid.client.selector.ServerSelectorStrategy; +import org.apache.druid.client.selector.TierSelectorStrategy; import org.apache.druid.discovery.NodeRole; +import org.apache.druid.guice.BrokerProcessingModule; import org.apache.druid.guice.ConditionalMultibind; import org.apache.druid.guice.ConfigProvider; +import org.apache.druid.guice.DruidProcessingModule; import org.apache.druid.guice.Jerseys; +import org.apache.druid.guice.JoinableFactoryModule; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; import org.apache.druid.guice.LazySingleton; +import org.apache.druid.guice.LegacyBrokerParallelMergeConfigModule; import org.apache.druid.guice.LifecycleModule; import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.guice.QueryRunnerFactoryModule; +import org.apache.druid.guice.QueryableModule; +import org.apache.druid.guice.SegmentWranglerModule; import org.apache.druid.guice.annotations.CoordinatorIndexingServiceDuty; import org.apache.druid.guice.annotations.CoordinatorMetadataStoreManagementDuty; import org.apache.druid.guice.annotations.EscalatedGlobal; @@ -73,8 +87,13 @@ import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.metadata.SegmentsMetadataManagerConfig; import org.apache.druid.metadata.SegmentsMetadataManagerProvider; +import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.apache.druid.server.ClientQuerySegmentWalker; import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.coordinator.DruidCoordinatorConfig; @@ -139,6 +158,7 @@ public class CliCoordinator extends ServerRunnable { private static final Logger log = new Logger(CliCoordinator.class); private static final String AS_OVERLORD_PROPERTY = "druid.coordinator.asOverlord.enabled"; + private static final String SEGMENT_METADATA_CACHE_ENABLED = "druid.coordinator.segmentMetadataCache.enabled"; private Properties properties; private boolean beOverlord; @@ -173,6 +193,12 @@ protected List getModules() List modules = new ArrayList<>(); modules.add(JettyHttpClientModule.global()); + modules.add(new LegacyBrokerParallelMergeConfigModule()); + modules.add(new QueryRunnerFactoryModule()); + modules.add(new SegmentWranglerModule()); + modules.add(new QueryableModule()); + modules.add(new BrokerProcessingModule()); + modules.add(new JoinableFactoryModule()); modules.add( new Module() @@ -190,6 +216,8 @@ public void configure(Binder binder) binder.bind(MetadataStorage.class).toProvider(MetadataStorageProvider.class); + binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); + JsonConfigProvider.bind(binder, SegmentsMetadataManagerConfig.CONFIG_PREFIX, SegmentsMetadataManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.lookups", LookupCoordinatorManagerConfig.class); @@ -200,6 +228,13 @@ public void configure(Binder binder) "druid.coordinator.balancer.cachingCost", CachingCostBalancerStrategyConfig.class ); + JsonConfigProvider.bind(binder, "druid.coordinator.segment", SegmentMetadataCacheConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.segment", BrokerSegmentWatcherConfig.class); binder.bind(RedirectFilter.class).in(LazySingleton.class); if (beOverlord) { @@ -221,13 +256,17 @@ public void configure(Binder binder) .in(ManageLifecycle.class); binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class); + binder.bind(CachingClusteredClient.class).in(LazySingleton.class); binder.bind(CoordinatorServerView.class); + binder.bind(TimelineServerView.class).to(CoordinatorServerView.class).in(LazySingleton.class); binder.bind(DruidCoordinator.class); LifecycleModule.register(binder, CoordinatorServerView.class); LifecycleModule.register(binder, MetadataStorage.class); LifecycleModule.register(binder, DruidCoordinator.class); + LifecycleModule.register(binder, SegmentMetadataCache.class); + binder.bind(JettyServerInitializer.class) .to(CoordinatorJettyServerInitializer.class); @@ -394,6 +433,12 @@ public static boolean isOverlord(Properties properties) return Boolean.parseBoolean(properties.getProperty(AS_OVERLORD_PROPERTY)); } + private boolean isSegmentMetadataCacheEnabled() + { + return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED)); + } + + private static class CoordinatorCustomDutyGroupsProvider implements Provider { private Properties props; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java index 484de5c4659b..272fe5030286 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java @@ -25,6 +25,7 @@ import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.LazySingleton; import org.apache.druid.sql.calcite.rule.ExtensionCalciteRuleProvider; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; /** * The module responsible for provide bindings for the Calcite Planner. @@ -41,7 +42,7 @@ public void configure(Binder binder) // It turns out that the order of the arguments above is misleading. // We're actually binding the class to the config prefix, not the other way around. JsonConfigProvider.bind(binder, CONFIG_BASE, PlannerConfig.class); - JsonConfigProvider.bind(binder, CONFIG_BASE, SegmentMetadataCacheConfig.class); + JsonConfigProvider.bind(binder, CONFIG_BASE, BrokerSegmentMetadataCacheConfig.class); binder.bind(PlannerFactory.class).in(LazySingleton.class); binder.bind(DruidOperatorTable.class).in(LazySingleton.class); Multibinder.newSetBinder(binder, ExtensionCalciteRuleProvider.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfig.java deleted file mode 100644 index dc4d94b78b39..000000000000 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfig.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.sql.calcite.planner; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.sql.calcite.schema.SegmentMetadataCache; -import org.joda.time.Period; - -/** - * Configuration properties for the Broker-side cache of segment metadata - * used to infer datasources for SQL. This class shares the same config root - * as {@link PlannerConfig} to maintain backward compatibility for when - * the properties here resided in {@code PlannerConfig}. - */ -public class SegmentMetadataCacheConfig -{ - @JsonProperty - private boolean awaitInitializationOnStart = true; - - @JsonProperty - private boolean metadataSegmentCacheEnable = false; - - @JsonProperty - private long metadataSegmentPollPeriod = 60000; - - @JsonProperty - private Period metadataRefreshPeriod = new Period("PT1M"); - - @JsonProperty - private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = - new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); - - public static SegmentMetadataCacheConfig create() - { - return new SegmentMetadataCacheConfig(); - } - - public static SegmentMetadataCacheConfig create( - String metadataRefreshPeriod - ) - { - SegmentMetadataCacheConfig config = new SegmentMetadataCacheConfig(); - config.metadataRefreshPeriod = new Period(metadataRefreshPeriod); - return config; - } - - public boolean isMetadataSegmentCacheEnable() - { - return metadataSegmentCacheEnable; - } - - public Period getMetadataRefreshPeriod() - { - return metadataRefreshPeriod; - } - - public boolean isAwaitInitializationOnStart() - { - return awaitInitializationOnStart; - } - - public long getMetadataSegmentPollPeriod() - { - return metadataSegmentPollPeriod; - } - - public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() - { - return metadataColumnTypeMergePolicy; - } - - @Override - public String toString() - { - return "SegmentCacheConfig{" + - "metadataRefreshPeriod=" + metadataRefreshPeriod + - ", metadataSegmentCacheEnable=" + metadataSegmentCacheEnable + - ", metadataSegmentPollPeriod=" + metadataSegmentPollPeriod + - ", awaitInitializationOnStart=" + awaitInitializationOnStart + - ", metadataColumnTypeMergePolicy=" + metadataColumnTypeMergePolicy + - '}'; - } -} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java new file mode 100644 index 000000000000..0224841a6cbc --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -0,0 +1,75 @@ +package org.apache.druid.sql.calcite.schema; + +import com.google.inject.Inject; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.security.Escalator; +import org.apache.druid.sql.calcite.table.DatasourceTable; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * + */ +@ManageLifecycle +public class BrokerSegmentMetadataCache extends SegmentMetadataCache +{ + private static final EmittingLogger log = new EmittingLogger(SegmentMetadataCache.class); + private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + + private final ConcurrentMap tables = new ConcurrentHashMap<>(); + + @Inject + public BrokerSegmentMetadataCache( + QueryLifecycleFactory queryLifecycleFactory, + TimelineServerView serverView, + SegmentMetadataCacheConfig config, + Escalator escalator, + InternalQueryConfig internalQueryConfig, + ServiceEmitter emitter, + PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder + ) + { + super( + queryLifecycleFactory, + serverView, + config, + escalator, + internalQueryConfig, + emitter + ); + this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; + } + + @Override + public void rebuildDatasource(String dataSource) + { + final DatasourceSchema druidTable = buildDruidTable(dataSource); + if (druidTable == null) { + log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + tables.remove(dataSource); + return; + } + final DatasourceTable.PhysicalDatasourceMetadata physicalDatasourceMetadata = + physicalDatasourceMetadataBuilder.build(dataSource, druidTable.getRowSignature()); + final DatasourceTable.PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); + if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + } else { + log.info("[%s] signature is unchanged.", dataSource); + } + } + + public DatasourceTable.PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) + { + return tables.get(name); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java new file mode 100644 index 000000000000..d7a059710893 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -0,0 +1,31 @@ +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; + +public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig +{ + @JsonProperty + private boolean metadataSegmentCacheEnable = false; + + @JsonProperty + private long metadataSegmentPollPeriod = 60000; + + @JsonProperty + private boolean useSegmentMetadataCache = false; + + public boolean isMetadataSegmentCacheEnable() + { + return metadataSegmentCacheEnable; + } + + public long getMetadataSegmentPollPeriod() + { + return metadataSegmentPollPeriod; + } + + public boolean isUseSegmentMetadataCache() + { + return useSegmentMetadataCache; + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java new file mode 100644 index 000000000000..cf92bba0eb69 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.inject.Inject; +import org.apache.druid.client.BrokerSegmentWatcherConfig; +import org.apache.druid.client.BrokerServerView; +import org.apache.druid.client.DataSegmentInterner; +import org.apache.druid.client.JsonParserIterator; +import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.client.selector.ServerSelector; +import org.apache.druid.common.guava.FutureUtils; +import org.apache.druid.concurrent.LifecycleLock; +import org.apache.druid.discovery.DruidLeaderClient; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.lifecycle.LifecycleStop; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.SegmentStatusInCluster; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * This class polls the Coordinator in background to keep the latest published segments. + * Provides {@link #getSegmentMetadata()} for others to get segments in metadata store. + * + * The difference between this class and {@link SegmentsMetadataManager} is that this class resides + * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In + * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the + * currently leading Coordinator via HTTP queries. + */ +@ManageLifecycle +public class BrokerSegmentMetadataView +{ + private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataView.class); + + private final CoordinatorClient coordinatorClient; + private final BrokerSegmentWatcherConfig segmentWatcherConfig; + private final DruidLeaderClient druidLeaderClient; + private final ObjectMapper objectMapper; + + private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + + private final boolean isMetadataSegmentCacheEnabled; + + private final boolean useSegmentMetadataCache; + + /** + * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and + * sys.segments queries return the segments in sorted order based on segmentId. + * + * Volatile since this reference is reassigned in {@code poll()} and then read in {@code getPublishedSegments()} + * from other threads. + */ + @MonotonicNonNull + private volatile ImmutableSortedSet segmentMetadata = null; + + private volatile Map datasourceSchemaMap = null; + + private final BrokerServerView brokerServerView; + + private final BrokerSegmentMetadataCache segmentMetadataCache; + + /** + * Caches the replication factor for segment IDs. In case of coordinator restarts or leadership re-elections, + * the coordinator API returns `null` replication factor until load rules are evaluated. + * The cache can be used during these periods to continue serving the previously fetched values. + */ + private final Cache segmentIdToReplicationFactor; + private final ScheduledExecutorService scheduledExec; + private final long pollPeriodInMS; + private final LifecycleLock lifecycleLock = new LifecycleLock(); + private final CountDownLatch segmentMetadataCachePopulated = new CountDownLatch(1); + + @Inject + public BrokerSegmentMetadataView( + final @Coordinator DruidLeaderClient druidLeaderClient, + final ObjectMapper objectMapper, + final CoordinatorClient coordinatorClient, + final BrokerSegmentWatcherConfig segmentWatcherConfig, + final BrokerSegmentMetadataCacheConfig config, + final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder, + final BrokerServerView brokerServerView, + final BrokerSegmentMetadataCache segmentMetadataCache + ) + { + Preconditions.checkNotNull(config, "SegmentMetadataCacheConfig"); + this.druidLeaderClient = druidLeaderClient; + this.objectMapper = objectMapper; + this.coordinatorClient = coordinatorClient; + this.segmentWatcherConfig = segmentWatcherConfig; + this.isMetadataSegmentCacheEnabled = config.isMetadataSegmentCacheEnable(); + this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); + this.scheduledExec = Execs.scheduledSingleThreaded("MetadataSegmentView-Cache--%d"); + this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() + .expireAfterAccess(10, TimeUnit.MINUTES) + .build(); + this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; + this.useSegmentMetadataCache = config.isUseSegmentMetadataCache(); + this.brokerServerView = brokerServerView; + this.segmentMetadataCache = segmentMetadataCache; + } + + @LifecycleStart + public void start() + { + if (!lifecycleLock.canStart()) { + throw new ISE("can't start."); + } + try { + if (isMetadataSegmentCacheEnabled || !useSegmentMetadataCache) { + scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); + } + lifecycleLock.started(); + log.info("MetadataSegmentView Started. Configs isMetadataSegmentCacheEnabled [%s], useSegmentMetadataCache [%s]", + isMetadataSegmentCacheEnabled, useSegmentMetadataCache); + } + finally { + lifecycleLock.exitStart(); + } + } + + @LifecycleStop + public void stop() + { + if (!lifecycleLock.canStop()) { + throw new ISE("can't stop."); + } + log.info("MetadataSegmentView is stopping."); + if (isMetadataSegmentCacheEnabled) { + scheduledExec.shutdown(); + } + log.info("MetadataSegmentView Stopped."); + } + + private void pollSegmentMetadata() + { + log.info("Polling segment metadata from coordinator"); + + segmentMetadata = fetchSegmentMetadata(); + segmentMetadataCachePopulated.countDown(); + } + + private ImmutableSortedSet fetchSegmentMetadata() + { + final Iterator metadataSegments = + querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); + + final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); + while (metadataSegments.hasNext()) { + final SegmentStatusInCluster segment = metadataSegments.next(); + final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); + Integer replicationFactor = segment.getReplicationFactor(); + if (replicationFactor == null) { + replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); + } else { + segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); + } + final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( + interned, + segment.isOvershadowed(), + replicationFactor, + segment.getNumRows(), + segment.isRealtime(), + segment.isPublished() + ); + builder.add(segmentStatusInCluster); + } + return builder.build(); + } + + private void pollDatasourceSchema() + { + log.info("Polling datasource schema from coordinator."); + Set datasources = useSegmentMetadataCache ? segmentMetadataCache.getDatasourceNames() : brokerServerView.getDatasourceNames(); + + Map physicalDatasourceMetadataMap = new HashMap<>(); + + for (List partition : Iterables.partition(datasources, 10)) { + List datasourceSchemas = FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema( + partition), true); + + for (DatasourceSchema datasourceSchema : datasourceSchemas) { + physicalDatasourceMetadataMap.put( + datasourceSchema.getDatasource(), + physicalDatasourceMetadataBuilder.build(datasourceSchema.getDatasource(), datasourceSchema.getRowSignature())); + } + } + + this.datasourceSchemaMap = physicalDatasourceMetadataMap; + } + + ImmutableSortedSet getSegmentMetadata() + { + if (isMetadataSegmentCacheEnabled) { + Uninterruptibles.awaitUninterruptibly(segmentMetadataCachePopulated); + return segmentMetadata; + } else { + return fetchSegmentMetadata(); + } + } + + protected SystemSchema.SegmentsTable.SegmentTableView getSegmentTableView() { + ImmutableSortedSet allSegmentMetadata = getSegmentMetadata(); + + final ImmutableSortedSet.Builder publishedSegmentBuilder = ImmutableSortedSet.naturalOrder(); + + Map availableSegmentMetadataMap; + + if (useSegmentMetadataCache) { + availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); + + for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { + if (segmentStatusInCluster.isPublished()) { + publishedSegmentBuilder.add(segmentStatusInCluster); + } + } + } else { + // build available segment metadata map by combining stuff from brokerServerView and numRows from data returned from coordinator + availableSegmentMetadataMap = new HashMap<>(); + Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); + for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { + if (segmentStatusInCluster.isPublished()) { + publishedSegmentBuilder.add(segmentStatusInCluster); + } + SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); + if (!brokerSegmentMetadata.containsKey(segmentId)) { + continue; + } + ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); + + AvailableSegmentMetadata availableSegmentMetadata = + AvailableSegmentMetadata.builder( + segmentStatusInCluster.getDataSegment(), + segmentStatusInCluster.isRealtime(), + Sets.newHashSet(serverSelector.getAllServers()), + null, + segmentStatusInCluster.getNumRows() == null ? -1 : segmentStatusInCluster.getNumRows() + ) + .build(); + + availableSegmentMetadataMap.put(segmentId, availableSegmentMetadata); + + } + } + + log.info("Logging Segment table view. availableSmMap [%s], published segments [%s]", availableSegmentMetadataMap, publishedSegmentBuilder); + return new SystemSchema.SegmentsTable.SegmentTableView(availableSegmentMetadataMap, publishedSegmentBuilder.build()); + } + + protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) + { + return useSegmentMetadataCache ? segmentMetadataCache.getPhysicalDatasourceMetadata(name) : datasourceSchemaMap.get(name); + } + + protected Set getDatasourceNames() + { + return useSegmentMetadataCache ? segmentMetadataCache.getDatasourceNames() : datasourceSchemaMap.keySet(); + } + + // Note that coordinator must be up to get segments + private JsonParserIterator querySegmentMetadata( + Set watchedDataSources + ) + { + String query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"; + if (watchedDataSources != null && !watchedDataSources.isEmpty()) { + log.debug( + "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); + final StringBuilder sb = new StringBuilder(); + for (String ds : watchedDataSources) { + sb.append("datasources=").append(ds).append("&"); + } + sb.setLength(sb.length() - 1); + query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&" + sb; + } + + return SystemSchema.getThingsFromLeaderNode( + query, + new TypeReference() + { + }, + druidLeaderClient, + objectMapper + ); + } + + private class PollTask implements Runnable + { + @Override + public void run() + { + long delayMS = pollPeriodInMS; + try { + final long pollStartTime = System.nanoTime(); + if (isMetadataSegmentCacheEnabled) { + pollSegmentMetadata(); + } + if (!useSegmentMetadataCache) { + pollDatasourceSchema(); + } + final long pollEndTime = System.nanoTime(); + final long pollTimeNS = pollEndTime - pollStartTime; + final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); + delayMS = Math.max(pollPeriodInMS - pollTimeMS, 0); + } + catch (Exception e) { + log.makeAlert(e, "Problem polling Coordinator.").emit(); + } + finally { + if (!Thread.currentThread().isInterrupted()) { + scheduledExec.schedule(new PollTask(), delayMS, TimeUnit.MILLISECONDS); + } + } + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index e9c894c91527..8b7d885ed64b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -19,7 +19,9 @@ package org.apache.druid.sql.calcite.schema; +import com.google.common.base.Preconditions; import com.google.inject.Binder; +import com.google.inject.Inject; import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; @@ -29,6 +31,8 @@ import org.apache.druid.guice.LifecycleModule; import org.apache.druid.sql.guice.SqlBindings; +import java.util.Properties; + /** * The module responsible for providing bindings to Calcite schemas. */ @@ -36,8 +40,16 @@ public class DruidCalciteSchemaModule implements Module { private static final String DRUID_SCHEMA_NAME = "druid"; private static final String INFORMATION_SCHEMA_NAME = "INFORMATION_SCHEMA"; + private static final String SEGMENT_METADATA_CACHE_DISABLED = "druid.sql.planner.disableSegmentMetadataCache"; static final String INCOMPLETE_SCHEMA = "INCOMPLETE_SCHEMA"; + private final Properties properties; + + public DruidCalciteSchemaModule(Properties properties) + { + this.properties = properties; + } + @Override public void configure(Binder binder) { @@ -49,8 +61,10 @@ public void configure(Binder binder) .toProvider(RootSchemaProvider.class) .in(Scopes.SINGLETON); - // SegmentMetadataCache needs to listen to changes for incoming segments - LifecycleModule.register(binder, SegmentMetadataCache.class); + // BrokerSegmentMetadataCache needs to listen to changes for incoming segments + if (!isSegmentMetadataCacheDisabled()) { + LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); + } binder.bind(DruidSchema.class).in(Scopes.SINGLETON); binder.bind(SystemSchema.class).in(Scopes.SINGLETON); binder.bind(InformationSchema.class).in(Scopes.SINGLETON); @@ -70,4 +84,9 @@ private DruidSchemaCatalog getRootSchema(@Named(INCOMPLETE_SCHEMA) DruidSchemaCa rootSchema.getRootSchema().add(INFORMATION_SCHEMA_NAME, informationSchema); return rootSchema; } + + private boolean isSegmentMetadataCacheDisabled() + { + return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_DISABLED)); + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index ec0b9cb2c116..28a6ab252516 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -28,15 +28,16 @@ public class DruidSchema extends AbstractTableSchema { - private final SegmentMetadataCache segmentCache; + private final BrokerSegmentMetadataView brokerSegmentMetadataView; private final DruidSchemaManager druidSchemaManager; @Inject public DruidSchema( - SegmentMetadataCache segmentCache, - final DruidSchemaManager druidSchemaManager) + final BrokerSegmentMetadataView brokerSegmentMetadataView, + final DruidSchemaManager druidSchemaManager + ) { - this.segmentCache = segmentCache; + this.brokerSegmentMetadataView = brokerSegmentMetadataView; if (druidSchemaManager != null && !(druidSchemaManager instanceof NoopDruidSchemaManager)) { this.druidSchemaManager = druidSchemaManager; } else { @@ -44,20 +45,15 @@ public DruidSchema( } } - protected SegmentMetadataCache cache() - { - return segmentCache; - } - @Override public Table getTable(String name) { if (druidSchemaManager != null) { return druidSchemaManager.getTable(name); } else { - DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentCache.getDatasource(name); + DatasourceTable.PhysicalDatasourceMetadata dsMetadata = brokerSegmentMetadataView.getDatasource(name); return dsMetadata == null ? null : new DatasourceTable(dsMetadata); - } +} } @Override @@ -66,7 +62,7 @@ public Set getTableNames() if (druidSchemaManager != null) { return druidSchemaManager.getTableNames(); } else { - return segmentCache.getDatasourceNames(); + return brokerSegmentMetadataView.getDatasourceNames(); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java deleted file mode 100644 index c3a1ab48f5ab..000000000000 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.sql.calcite.schema; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.util.concurrent.Uninterruptibles; -import com.google.inject.Inject; -import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.DataSegmentInterner; -import org.apache.druid.client.JsonParserIterator; -import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.concurrent.LifecycleLock; -import org.apache.druid.discovery.DruidLeaderClient; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.lifecycle.LifecycleStop; -import org.apache.druid.java.util.emitter.EmittingLogger; -import org.apache.druid.metadata.SegmentsMetadataManager; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.SegmentStatusInCluster; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * This class polls the Coordinator in background to keep the latest published segments. - * Provides {@link #getPublishedSegments()} for others to get segments in metadata store. - * - * The difference between this class and {@link SegmentsMetadataManager} is that this class resides - * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In - * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the - * currently leading Coordinator via HTTP queries. - */ -@ManageLifecycle -public class MetadataSegmentView -{ - private static final EmittingLogger log = new EmittingLogger(MetadataSegmentView.class); - - private final DruidLeaderClient coordinatorDruidLeaderClient; - private final ObjectMapper jsonMapper; - private final BrokerSegmentWatcherConfig segmentWatcherConfig; - - private final boolean isCacheEnabled; - /** - * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and - * sys.segments queries return the segments in sorted order based on segmentId. - * - * Volatile since this reference is reassigned in {@code poll()} and then read in {@code getPublishedSegments()} - * from other threads. - */ - @MonotonicNonNull - private volatile ImmutableSortedSet publishedSegments = null; - /** - * Caches the replication factor for segment IDs. In case of coordinator restarts or leadership re-elections, the coordinator API returns `null` replication factor until load rules are evaluated. - * The cache can be used during these periods to continue serving the previously fetched values. - */ - private final Cache segmentIdToReplicationFactor; - private final ScheduledExecutorService scheduledExec; - private final long pollPeriodInMS; - private final LifecycleLock lifecycleLock = new LifecycleLock(); - private final CountDownLatch cachePopulated = new CountDownLatch(1); - - @Inject - public MetadataSegmentView( - final @Coordinator DruidLeaderClient druidLeaderClient, - final ObjectMapper jsonMapper, - final BrokerSegmentWatcherConfig segmentWatcherConfig, - final SegmentMetadataCacheConfig config - ) - { - Preconditions.checkNotNull(config, "SegmentMetadataCacheConfig"); - this.coordinatorDruidLeaderClient = druidLeaderClient; - this.jsonMapper = jsonMapper; - this.segmentWatcherConfig = segmentWatcherConfig; - this.isCacheEnabled = config.isMetadataSegmentCacheEnable(); - this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); - this.scheduledExec = Execs.scheduledSingleThreaded("MetadataSegmentView-Cache--%d"); - this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() - .expireAfterAccess(10, TimeUnit.MINUTES) - .build(); - } - - @LifecycleStart - public void start() - { - if (!lifecycleLock.canStart()) { - throw new ISE("can't start."); - } - try { - if (isCacheEnabled) { - scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); - } - lifecycleLock.started(); - log.info("MetadataSegmentView Started."); - } - finally { - lifecycleLock.exitStart(); - } - } - - @LifecycleStop - public void stop() - { - if (!lifecycleLock.canStop()) { - throw new ISE("can't stop."); - } - log.info("MetadataSegmentView is stopping."); - if (isCacheEnabled) { - scheduledExec.shutdown(); - } - log.info("MetadataSegmentView Stopped."); - } - - private void poll() - { - log.info("polling published segments from coordinator"); - final JsonParserIterator metadataSegments = getMetadataSegments( - coordinatorDruidLeaderClient, - jsonMapper, - segmentWatcherConfig.getWatchedDataSources() - ); - - final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); - while (metadataSegments.hasNext()) { - final SegmentStatusInCluster segment = metadataSegments.next(); - final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); - Integer replicationFactor = segment.getReplicationFactor(); - if (replicationFactor == null) { - replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); - } else { - segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); - } - final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( - interned, - segment.isOvershadowed(), - replicationFactor - ); - builder.add(segmentStatusInCluster); - } - publishedSegments = builder.build(); - cachePopulated.countDown(); - } - - Iterator getPublishedSegments() - { - if (isCacheEnabled) { - Uninterruptibles.awaitUninterruptibly(cachePopulated); - return publishedSegments.iterator(); - } else { - return getMetadataSegments( - coordinatorDruidLeaderClient, - jsonMapper, - segmentWatcherConfig.getWatchedDataSources() - ); - } - } - - // Note that coordinator must be up to get segments - private JsonParserIterator getMetadataSegments( - DruidLeaderClient coordinatorClient, - ObjectMapper jsonMapper, - Set watchedDataSources - ) - { - String query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"; - if (watchedDataSources != null && !watchedDataSources.isEmpty()) { - log.debug( - "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); - final StringBuilder sb = new StringBuilder(); - for (String ds : watchedDataSources) { - sb.append("datasources=").append(ds).append("&"); - } - sb.setLength(sb.length() - 1); - query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&" + sb; - } - - return SystemSchema.getThingsFromLeaderNode( - query, - new TypeReference() - { - }, - coordinatorClient, - jsonMapper - ); - } - - private class PollTask implements Runnable - { - @Override - public void run() - { - long delayMS = pollPeriodInMS; - try { - final long pollStartTime = System.nanoTime(); - poll(); - final long pollEndTime = System.nanoTime(); - final long pollTimeNS = pollEndTime - pollStartTime; - final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); - delayMS = Math.max(pollPeriodInMS - pollTimeMS, 0); - } - catch (Exception e) { - log.makeAlert(e, "Problem polling Coordinator.").emit(); - } - finally { - if (!Thread.currentThread().isInterrupted()) { - scheduledExec.schedule(new PollTask(), delayMS, TimeUnit.MILLISECONDS); - } - } - } - } - -} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java new file mode 100644 index 000000000000..c9760aeb0cf4 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -0,0 +1,42 @@ +package org.apache.druid.sql.calcite.schema; + +import com.google.inject.Inject; +import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.sql.calcite.table.DatasourceTable; + +public class PhysicalDatasourceMetadataBuilder +{ + private final JoinableFactory joinableFactory; + private final SegmentManager segmentManager; + + @Inject + public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, SegmentManager segmentManager) + { + this.joinableFactory = joinableFactory; + this.segmentManager = segmentManager; + } + + DatasourceTable.PhysicalDatasourceMetadata build(String dataSource, RowSignature rowSignature) + { + final TableDataSource tableDataSource; + + // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing + // in the segment cache, which in this case belongs to the broker meaning only broadcast segments live here) + // to be joinable, it must be possibly joinable according to the factory. we only consider broadcast datasources + // at this time, and isGlobal is currently strongly coupled with joinable, so only make a global table datasource + // if also joinable + final GlobalTableDataSource maybeGlobal = new GlobalTableDataSource(dataSource); + final boolean isJoinable = joinableFactory.isDirectlyJoinable(maybeGlobal); + final boolean isBroadcast = segmentManager.getDataSourceNames().contains(dataSource); + if (isBroadcast && isJoinable) { + tableDataSource = maybeGlobal; + } else { + tableDataSource = new TableDataSource(dataSource); + } + return new DatasourceTable.PhysicalDatasourceMetadata(tableDataSource, rowSignature, isJoinable, isBroadcast); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index db4a10af81c7..f1063c521ef0 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -27,6 +27,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -66,6 +67,7 @@ import org.apache.druid.java.util.http.client.Request; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; @@ -212,7 +214,7 @@ public class SystemSchema extends AbstractSchema @Inject public SystemSchema( final DruidSchema druidSchema, - final MetadataSegmentView metadataView, + final BrokerSegmentMetadataView metadataView, final TimelineServerView serverView, final FilteredServerInventoryView serverInventoryView, final AuthorizerMapper authorizerMapper, @@ -246,11 +248,11 @@ static class SegmentsTable extends AbstractTable implements ScannableTable private final DruidSchema druidSchema; private final ObjectMapper jsonMapper; private final AuthorizerMapper authorizerMapper; - private final MetadataSegmentView metadataView; + private final BrokerSegmentMetadataView metadataView; public SegmentsTable( DruidSchema druidSchemna, - MetadataSegmentView metadataView, + BrokerSegmentMetadataView metadataView, ObjectMapper jsonMapper, AuthorizerMapper authorizerMapper ) @@ -276,15 +278,16 @@ public TableType getJdbcTableType() @Override public Enumerable scan(DataContext root) { - //get available segments from druidSchema + final SegmentTableView segmentTableView = metadataView.getSegmentTableView(); + final Map availableSegmentMetadata = - druidSchema.cache().getSegmentMetadataSnapshot(); + segmentTableView.getAvailableSegmentMetadata(); final Iterator> availableSegmentEntries = availableSegmentMetadata.entrySet().iterator(); // in memory map to store segment data from available segments final Map partialSegmentDataMap = - Maps.newHashMapWithExpectedSize(druidSchema.cache().getTotalSegments()); + Maps.newHashMapWithExpectedSize(segmentTableView.totalSegmentCount()); for (AvailableSegmentMetadata h : availableSegmentMetadata.values()) { PartialSegmentData partialSegmentData = new PartialSegmentData(IS_AVAILABLE_TRUE, h.isRealtime(), h.getNumReplicas(), h.getNumRows()); @@ -293,9 +296,9 @@ public Enumerable scan(DataContext root) // Get published segments from metadata segment cache (if enabled in SQL planner config), else directly from // Coordinator. - final Iterator metadataStoreSegments = metadataView.getPublishedSegments(); + final Iterator metadataStoreSegments = segmentTableView.getPublishedSegments(); - final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(druidSchema.cache().getTotalSegments()); + final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(segmentTableView.totalSegmentCount()); final FluentIterable publishedSegments = FluentIterable .from(() -> getAuthorizedPublishedSegments(metadataStoreSegments, root)) @@ -439,7 +442,40 @@ private Iterator> getAuthorizedAvaila return authorizedSegments.iterator(); } - private static class PartialSegmentData + protected static class SegmentTableView + { + private final Map availableSegmentMetadata; + private final Iterator publishedSegments; + + private final int totalSegmentsCount; + + public SegmentTableView( + Map availableSegmentMetadata, + ImmutableSortedSet publishedSegments + ) + { + this.availableSegmentMetadata = availableSegmentMetadata; + this.publishedSegments = publishedSegments.iterator(); + this.totalSegmentsCount = availableSegmentMetadata.size(); + } + + public Map getAvailableSegmentMetadata() + { + return availableSegmentMetadata; + } + + public Iterator getPublishedSegments() + { + return publishedSegments; + } + + public int totalSegmentCount() + { + return totalSegmentsCount; + } + } + + protected static class PartialSegmentData { private final long isAvailable; private final long isRealtime; diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 56d0d2d5d41f..03a774a05b61 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -104,7 +104,7 @@ public void configure(Binder binder) binder.bind(TableDefnRegistry.class).in(LazySingleton.class); - binder.install(new DruidCalciteSchemaModule()); + binder.install(new DruidCalciteSchemaModule(props)); binder.install(new CalcitePlannerModule()); binder.install(new SqlAggregationModule()); binder.install(new DruidViewModule()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java index 858f2e5d8ac9..29de2639ae4a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java @@ -24,7 +24,8 @@ import org.apache.druid.guice.GuiceInjectors; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; -import org.apache.druid.sql.calcite.schema.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.joda.time.Period; import org.junit.Assert; import org.junit.Test; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index d10bd2e227b8..c1a0d5047ea7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -21,8 +21,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; @@ -30,7 +31,7 @@ import org.apache.druid.server.SegmentManager; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; @@ -60,7 +61,7 @@ public void testInitializationWithNoData() throws Exception new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java index 10964800bc43..d34bed3ccf73 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.DruidServer; @@ -42,6 +42,9 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.metadata.PhysicalDatasourceMetadata; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.TableDataSource; @@ -140,7 +143,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -250,7 +253,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java index e7eb01a59f95..97534f2f81d9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java @@ -33,7 +33,7 @@ import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.easymock.EasyMock; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java index b54149054895..ae5f14aab8c6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java @@ -30,7 +30,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; @@ -40,6 +40,8 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; @@ -70,7 +72,7 @@ import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.sql.calcite.table.DruidTable; import org.apache.druid.sql.calcite.util.CalciteTests; @@ -299,7 +301,7 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheCon ), config, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -337,7 +339,7 @@ public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedE ), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -708,7 +710,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -751,7 +753,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -798,7 +800,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -842,7 +844,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -883,7 +885,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -941,7 +943,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -1002,7 +1004,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -1037,7 +1039,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -1085,7 +1087,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -1288,11 +1290,11 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception String brokerInternalQueryConfigJson = "{\"context\": { \"priority\": 5} }"; TestHelper.makeJsonMapper(); - BrokerInternalQueryConfig brokerInternalQueryConfig = MAPPER.readValue( + InternalQueryConfig internalQueryConfig = MAPPER.readValue( MAPPER.writeValueAsString( - MAPPER.readValue(brokerInternalQueryConfigJson, BrokerInternalQueryConfig.class) + MAPPER.readValue(brokerInternalQueryConfigJson, InternalQueryConfig.class) ), - BrokerInternalQueryConfig.class + InternalQueryConfig.class ); DataSegment segment = newSegment("test", 0); @@ -1327,7 +1329,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception ), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - brokerInternalQueryConfig, + internalQueryConfig, new NoopServiceEmitter() ); @@ -1463,7 +1465,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), emitter ) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 82eae68c3321..52439c33d6e9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -36,7 +36,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.ImmutableDruidDataSource; @@ -64,6 +64,7 @@ import org.apache.druid.java.util.http.client.response.HttpResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -93,7 +94,7 @@ import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.server.security.ResourceType; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable; import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.sql.calcite.util.CalciteTestBase; @@ -165,7 +166,7 @@ public class SystemSchemaTest extends CalciteTestBase private AuthorizerMapper authMapper; private static QueryRunnerFactoryConglomerate conglomerate; private static Closer resourceCloser; - private MetadataSegmentView metadataView; + private BrokerSegmentMetadataView metadataView; private DruidNodeDiscoveryProvider druidNodeDiscoveryProvider; private FilteredServerInventoryView serverInventoryView; @@ -258,13 +259,13 @@ public void setUp() throws Exception new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ); cache.start(); cache.awaitInitialization(); druidSchema = new DruidSchema(cache, null); - metadataView = EasyMock.createMock(MetadataSegmentView.class); + metadataView = EasyMock.createMock(BrokerSegmentMetadataView.class); druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class); schema = new SystemSchema( @@ -567,14 +568,14 @@ public void testSegmentsTable() throws Exception { final SegmentsTable segmentsTable = new SegmentsTable(druidSchema, metadataView, new ObjectMapper(), authMapper); final Set publishedSegments = new HashSet<>(Arrays.asList( - new SegmentStatusInCluster(publishedCompactedSegment1, true, 2), - new SegmentStatusInCluster(publishedCompactedSegment2, false, 0), - new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2), - new SegmentStatusInCluster(segment1, true, 2), - new SegmentStatusInCluster(segment2, false, 0) + new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 2L, 0L, true), + new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 2L, 0L, true), + new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 2L, 0L, true), + new SegmentStatusInCluster(segment1, true, 2, 2L, 0L, true), + new SegmentStatusInCluster(segment2, false, 0, 2L, 0L, true) )); - EasyMock.expect(metadataView.getPublishedSegments()).andReturn(publishedSegments.iterator()).once(); + EasyMock.expect(metadataView.getSegmentMetadata()).andReturn(publishedSegments.iterator()).once(); EasyMock.replay(client, request, responseHolder, responseHandler, metadataView); DataContext dataContext = createDataContext(Users.SUPER); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index bab3e0957e8f..f182bd5dab58 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -68,12 +68,12 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; -import org.apache.druid.sql.calcite.schema.MetadataSegmentView; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.timeline.DataSegment; import org.joda.time.Duration; @@ -372,7 +372,7 @@ public ListenableFuture findCurrentLeader() return new SystemSchema( druidSchema, - new MetadataSegmentView( + new BrokerSegmentMetadataView( druidLeaderClient, getJsonMapper(), new BrokerSegmentWatcherConfig(), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 63235a76c7e1..1e2b9aab25bf 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -25,7 +25,7 @@ import com.google.inject.Injector; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.schema.SchemaPlus; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.guice.ExpressionModule; import org.apache.druid.java.util.emitter.core.NoopEmitter; import org.apache.druid.java.util.emitter.service.ServiceEmitter; @@ -55,7 +55,7 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; @@ -68,7 +68,7 @@ import org.apache.druid.sql.calcite.schema.NamedSystemSchema; import org.apache.druid.sql.calcite.schema.NamedViewSchema; import org.apache.druid.sql.calcite.schema.NoopDruidSchemaManager; -import org.apache.druid.sql.calcite.schema.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.sql.calcite.schema.ViewSchema; import org.apache.druid.sql.calcite.view.ViewManager; @@ -84,6 +84,7 @@ public class QueryFrameworkUtils { + public static final String INFORMATION_SCHEMA_NAME = "INFORMATION_SCHEMA"; public static QueryLifecycleFactory createMockQueryLifecycleFactory( @@ -224,7 +225,7 @@ public Set getDataSourceNames() createDefaultJoinableFactory(injector), SegmentMetadataCacheConfig.create(), CalciteTests.TEST_AUTHENTICATOR_ESCALATOR, - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ); @@ -273,5 +274,5 @@ public static DruidOperatorTable createOperatorTable(final Injector injector) public static LookupSchema createMockLookupSchema(final Injector injector) { return new LookupSchema(injector.getInstance(LookupExtractorFactoryContainerProvider.class)); - } + } **/ } From d1c0dca1494f6fdbeff74cd17170232f8c9450ea Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 28 Aug 2023 18:21:45 +0530 Subject: [PATCH 02/82] refactor CoordinatorServerView --- .../druid/client/CoordinatorServerView.java | 478 +----------------- .../client/CoordinatorServerViewImpl.java | 224 ++++++++ .../TimelineAwareCoordinatorServerView.java | 378 ++++++++++++++ ...oordinatorServerViewImplImplImplTest.java} | 8 +- 4 files changed, 612 insertions(+), 476 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java create mode 100644 server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java rename server/src/test/java/org/apache/druid/client/{CoordinatorServerViewTest.java => CoordinatorServerViewImplImplImplTest.java} (98%) diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index 45cdc6b11b91..b995ed70b7fa 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -1,481 +1,15 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.druid.client; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Iterables; -import com.google.common.collect.Ordering; -import com.google.inject.Inject; -import org.apache.druid.client.selector.QueryableDruidServer; -import org.apache.druid.client.selector.ServerSelector; -import org.apache.druid.client.selector.TierSelectorStrategy; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.guice.annotations.EscalatedClient; -import org.apache.druid.guice.annotations.Smile; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.client.InventoryView; +import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.query.DataSource; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryToolChestWarehouse; -import org.apache.druid.query.QueryWatcher; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.planning.DataSourceAnalysis; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; -import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.TimelineLookup; import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.PartitionChunk; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.function.Function; -import java.util.stream.Collectors; -/** - * ServerView of coordinator for the state of segments being loaded in the cluster. - */ -@ManageLifecycle -public class CoordinatorServerView implements InventoryView, TimelineServerView +public interface CoordinatorServerView extends InventoryView { - private static final Logger log = new Logger(CoordinatorServerView.class); - - private final Object lock = new Object(); - - private final Map segmentLoadInfos; - private final ConcurrentMap clients = new ConcurrentHashMap<>(); - private final Map selectors = new HashMap<>(); - private final Map> timelines2 = new HashMap<>(); - private final Map> timelines; - - private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); - - private final ServerInventoryView baseView; - private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; - - private final CountDownLatch initialized = new CountDownLatch(1); - private final ServiceEmitter emitter; - - private final TierSelectorStrategy tierSelectorStrategy; - - private final QueryToolChestWarehouse warehouse; - private final QueryWatcher queryWatcher; - private final ObjectMapper smileMapper; - private final HttpClient httpClient; - - @Inject - public CoordinatorServerView( - final QueryToolChestWarehouse warehouse, - final QueryWatcher queryWatcher, - final @Smile ObjectMapper smileMapper, - final @EscalatedClient HttpClient httpClient, - ServerInventoryView baseView, - CoordinatorSegmentWatcherConfig segmentWatcherConfig, - ServiceEmitter emitter, - TierSelectorStrategy tierSelectorStrategy - ) - { - this.baseView = baseView; - this.segmentWatcherConfig = segmentWatcherConfig; - this.emitter = emitter; - this.segmentLoadInfos = new HashMap<>(); - this.timelines = new HashMap<>(); - this.tierSelectorStrategy = tierSelectorStrategy; - this.warehouse = warehouse; - this.queryWatcher = queryWatcher; - this.smileMapper = smileMapper; - this.httpClient = httpClient; - - ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); - baseView.registerSegmentCallback( - exec, - new ServerView.SegmentCallback() - { - @Override - public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) - { - serverAddedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) - { - serverRemovedSegment2(server, segment); - serverRemovedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentViewInitialized() - { - initialized.countDown(); - runTimelineCallbacks(TimelineCallback::timelineInitialized); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - - baseView.registerServerRemovedCallback( - exec, - new ServerView.ServerRemovedCallback() - { - @Override - public ServerView.CallbackAction serverRemoved(DruidServer server) - { - removeServer(server); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - } - - @LifecycleStart - public void start() throws InterruptedException - { - if (segmentWatcherConfig.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); - initialized.await(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( - "serverview/init/time", - endMillis - startMillis - )); - } - } - - private void removeServer(DruidServer server) - { - for (DataSegment segment : server.iterateAllSegments()) { - serverAddedSegment2(server.getMetadata(), segment); - serverRemovedSegment(server.getMetadata(), segment); - } - } - - private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) - { - SegmentId segmentId = segment.getId(); - synchronized (lock) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - - SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - // servers escape the scope of this object so use ConcurrentSet - segmentLoadInfo = new SegmentLoadInfo(segment); - - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - if (timeline == null) { - timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - timelines.put(segment.getDataSource(), timeline); - } - - timeline.add( - segment.getInterval(), - segment.getVersion(), - segment.getShardSpec().createChunk(segmentLoadInfo) - ); - segmentLoadInfos.put(segmentId, segmentLoadInfo); - } - segmentLoadInfo.addServer(server); - } - runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); - } - - private void serverAddedSegment2(final DruidServerMetadata server, final DataSegment segment) - { - final SegmentId segmentId = segment.getId(); - synchronized (lock) { - // in theory we could probably just filter this to ensure we don't put ourselves in here, to make broker tree - // query topologies, but for now just skip all brokers, so we don't create some sort of wild infinite query - // loop... - if (!server.getType().equals(ServerType.BROKER)) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - ServerSelector selector = selectors.get(segmentId); - if (selector == null) { - selector = new ServerSelector(segment, tierSelectorStrategy); - - VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); - if (timeline == null) { - // broker needs to skip tombstones - timeline = new VersionedIntervalTimeline<>(Ordering.natural(), true); - timelines2.put(segment.getDataSource(), timeline); - } - - timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); - selectors.put(segmentId, selector); - } - - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); - if (inventoryValue == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping addition of segment[%s].", - server.getName(), - segmentId - ); - return; - } else { - queryableDruidServer = addServer(inventoryValue); - } - } - selector.addServerAndUpdateSegment(queryableDruidServer, segment); - } - // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it - runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); - } - } - - private QueryableDruidServer addServer(DruidServer server) - { - QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); - QueryableDruidServer exists = clients.put(server.getName(), retVal); - if (exists != null) { - log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); - } - - return retVal; - } - - private DirectDruidClient makeDirectClient(DruidServer server) - { - return new DirectDruidClient( - warehouse, - queryWatcher, - smileMapper, - httpClient, - server.getScheme(), - server.getHost(), - emitter - ); - } - - private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) - { - SegmentId segmentId = segment.getId(); - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - if (segmentLoadInfo.removeServer(server)) { - runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); - } - if (segmentLoadInfo.isEmpty()) { - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - segmentLoadInfos.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( - new SegmentLoadInfo( - segment - ) - ) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } else { - runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); - } - } - } - } - - private void serverRemovedSegment2(DruidServerMetadata server, DataSegment segment) - { - final SegmentId segmentId = segment.getId(); - final ServerSelector selector; - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - // we don't store broker segments here, but still run the callbacks for the segment being removed from the server - // since the broker segments are not stored on the timeline, do not fire segmentRemoved event - selector = selectors.get(segmentId); - if (selector == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping removal of segment[%s].", - server.getName(), - segmentId - ); - } else if (!selector.removeServer(queryableDruidServer)) { - log.warn( - "Asked to disassociate non-existant association between server[%s] and segment[%s]", - server, - segmentId - ); - } else { - // runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); - } - - if (selector.isEmpty()) { - VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); - selectors.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } else { - // runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); - } - } - } - } - - public VersionedIntervalTimeline getTimeline(DataSource dataSource) - { - String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (lock) { - return timelines.get(table); - } - } - - public Set getLoadedSegmentIds() - { - return segmentLoadInfos.keySet(); - } - - @Override - public DruidServer getInventoryValue(String serverKey) - { - return baseView.getInventoryValue(serverKey); - } - - @Override - public Collection getInventory() - { - return baseView.getInventory(); - } - - @Override - public boolean isStarted() - { - return baseView.isStarted(); - } - - @Override - public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) - { - return baseView.isSegmentLoadedByServer(serverKey, segment); - } - - private void runTimelineCallbacks(final Function function) - { - for (Map.Entry entry : timelineCallbacks.entrySet()) { - entry.getValue().execute( - () -> { - if (CallbackAction.UNREGISTER == function.apply(entry.getKey())) { - timelineCallbacks.remove(entry.getKey()); - } - } - ); - } - } - - @Override - public void registerTimelineCallback(Executor exec, TimelineCallback callback) - { - timelineCallbacks.put(callback, exec); - } - - @Override - public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) - { - throw new UnsupportedOperationException(); - } - - @Override - public void registerSegmentCallback(Executor exec, SegmentCallback callback) - { - baseView.registerSegmentCallback(exec, callback); - } - - @Override - public Optional> getTimeline(DataSourceAnalysis analysis) - { - final TableDataSource table = - analysis.getBaseTableDataSource() - .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); - - synchronized (lock) { - return Optional.ofNullable(timelines2.get(table.getName())); - } - } - - @Override - public List getDruidServers() - { - return clients.values().stream() - .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) - .collect(Collectors.toList()); - } - - @Override - public QueryRunner getQueryRunner(DruidServer server) - { - synchronized (lock) { - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - log.error("No QueryRunner found for server name[%s].", server.getName()); - return null; - } - return queryableDruidServer.getQueryRunner(); - } - } -} + VersionedIntervalTimeline getTimeline(DataSource dataSource); + Map getSegmentLoadInfos(); +} \ No newline at end of file diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java new file mode 100644 index 000000000000..cfcab66105bd --- /dev/null +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java @@ -0,0 +1,224 @@ +package org.apache.druid.client; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.inject.Inject; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.query.DataSource; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.PartitionChunk; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; + +/** + * ServerView of coordinator for the state of segments being loaded in the cluster. + */ +@ManageLifecycle +public class CoordinatorServerViewImpl implements CoordinatorServerView +{ + private static final Logger log = new Logger(Alpha.class); + + private final Object lock = new Object(); + + private final Map segmentLoadInfos; + private final Map> timelines; + + private final ServerInventoryView baseView; + private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; + + private final CountDownLatch initialized = new CountDownLatch(1); + private final ServiceEmitter emitter; + + @Inject + public CoordinatorServerViewImpl( + ServerInventoryView baseView, + CoordinatorSegmentWatcherConfig segmentWatcherConfig, + ServiceEmitter emitter + ) + { + this.baseView = baseView; + this.segmentWatcherConfig = segmentWatcherConfig; + this.emitter = emitter; + this.segmentLoadInfos = new HashMap<>(); + this.timelines = new HashMap<>(); + + ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); + baseView.registerSegmentCallback( + exec, + new ServerView.SegmentCallback() + { + @Override + public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + serverAddedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) + { + serverRemovedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentViewInitialized() + { + initialized.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + + baseView.registerServerRemovedCallback( + exec, + new ServerView.ServerRemovedCallback() + { + @Override + public ServerView.CallbackAction serverRemoved(DruidServer server) + { + removeServer(server); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + } + + @LifecycleStart + public void start() throws InterruptedException + { + if (segmentWatcherConfig.isAwaitInitializationOnStart()) { + final long startMillis = System.currentTimeMillis(); + log.info("%s waiting for initialization.", getClass().getSimpleName()); + initialized.await(); + final long endMillis = System.currentTimeMillis(); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); + emitter.emit(ServiceMetricEvent.builder().build( + "serverview/init/time", + endMillis - startMillis + )); + } + } + + private void removeServer(DruidServer server) + { + for (DataSegment segment : server.iterateAllSegments()) { + serverRemovedSegment(server.getMetadata(), segment); + } + } + + private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) + { + SegmentId segmentId = segment.getId(); + synchronized (lock) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + + SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + // servers escape the scope of this object so use ConcurrentSet + segmentLoadInfo = new SegmentLoadInfo(segment); + + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + if (timeline == null) { + timeline = new VersionedIntervalTimeline<>(Ordering.natural()); + timelines.put(segment.getDataSource(), timeline); + } + + timeline.add( + segment.getInterval(), + segment.getVersion(), + segment.getShardSpec().createChunk(segmentLoadInfo) + ); + segmentLoadInfos.put(segmentId, segmentLoadInfo); + } + segmentLoadInfo.addServer(server); + } + } + + private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) + { + SegmentId segmentId = segment.getId(); + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + segmentLoadInfo.removeServer(server); + if (segmentLoadInfo.isEmpty()) { + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + segmentLoadInfos.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( + new SegmentLoadInfo( + segment + ) + ) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } + } + } + } + + @Override + public VersionedIntervalTimeline getTimeline(DataSource dataSource) + { + String table = Iterables.getOnlyElement(dataSource.getTableNames()); + synchronized (lock) { + return timelines.get(table); + } + } + + @Override + public Map getSegmentLoadInfos() + { + return segmentLoadInfos; + } + + @Override + public DruidServer getInventoryValue(String serverKey) + { + return baseView.getInventoryValue(serverKey); + } + + @Override + public Collection getInventory() + { + return baseView.getInventory(); + } + + @Override + public boolean isStarted() + { + return baseView.isStarted(); + } + + @Override + public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) + { + return baseView.isSegmentLoadedByServer(serverKey, segment); + } +} diff --git a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java new file mode 100644 index 000000000000..a58c2e18957f --- /dev/null +++ b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java @@ -0,0 +1,378 @@ +package org.apache.druid.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.inject.Inject; +import org.apache.druid.client.selector.QueryableDruidServer; +import org.apache.druid.client.selector.ServerSelector; +import org.apache.druid.client.selector.TierSelectorStrategy; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.guice.annotations.EscalatedClient; +import org.apache.druid.guice.annotations.Smile; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.QueryWatcher; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.planning.DataSourceAnalysis; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.TimelineLookup; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.PartitionChunk; +import org.apache.druid.utils.CollectionUtils; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.function.Function; +import java.util.stream.Collectors; + +@ManageLifecycle +public class TimelineAwareCoordinatorServerView implements CoordinatorServerView, TimelineServerView +{ + private static final Logger log = new Logger(TimelineAwareCoordinatorServerView.class); + + private final Object lock = new Object(); + + private final ConcurrentMap clients = new ConcurrentHashMap<>(); + private final Map selectors = new HashMap<>(); + private final Map> timelines = new HashMap<>(); + + private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); + + private final ServerInventoryView baseView; + private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; + + private final CountDownLatch initialized = new CountDownLatch(1); + private final ServiceEmitter emitter; + + private final TierSelectorStrategy tierSelectorStrategy; + + private final QueryToolChestWarehouse warehouse; + private final QueryWatcher queryWatcher; + private final ObjectMapper smileMapper; + private final HttpClient httpClient; + + @Inject + public TimelineAwareCoordinatorServerView( + final QueryToolChestWarehouse warehouse, + final QueryWatcher queryWatcher, + final @Smile ObjectMapper smileMapper, + final @EscalatedClient HttpClient httpClient, + ServerInventoryView baseView, + CoordinatorSegmentWatcherConfig segmentWatcherConfig, + ServiceEmitter emitter, + TierSelectorStrategy tierSelectorStrategy + ) + { + this.baseView = baseView; + this.segmentWatcherConfig = segmentWatcherConfig; + this.emitter = emitter; + this.tierSelectorStrategy = tierSelectorStrategy; + this.warehouse = warehouse; + this.queryWatcher = queryWatcher; + this.smileMapper = smileMapper; + this.httpClient = httpClient; + + ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); + baseView.registerSegmentCallback( + exec, + new ServerView.SegmentCallback() + { + @Override + public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + serverAddedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) + { + serverRemovedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentViewInitialized() + { + initialized.countDown(); + runTimelineCallbacks(TimelineCallback::timelineInitialized); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + + baseView.registerServerRemovedCallback( + exec, + new ServerView.ServerRemovedCallback() + { + @Override + public ServerView.CallbackAction serverRemoved(DruidServer server) + { + removeServer(server); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + } + + @LifecycleStart + public void start() throws InterruptedException + { + if (segmentWatcherConfig.isAwaitInitializationOnStart()) { + final long startMillis = System.currentTimeMillis(); + log.info("%s waiting for initialization.", getClass().getSimpleName()); + initialized.await(); + final long endMillis = System.currentTimeMillis(); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); + emitter.emit(ServiceMetricEvent.builder().build( + "serverview/init/time", + endMillis - startMillis + )); + } + } + + private void removeServer(DruidServer server) + { + for (DataSegment segment : server.iterateAllSegments()) { + serverRemovedSegment(server.getMetadata(), segment); + } + } + + private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + synchronized (lock) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + ServerSelector selector = selectors.get(segmentId); + if (selector == null) { + selector = new ServerSelector(segment, tierSelectorStrategy); + + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + if (timeline == null) { + // broker needs to skip tombstones + timeline = new VersionedIntervalTimeline<>(Ordering.natural()); + timelines.put(segment.getDataSource(), timeline); + } + + timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); + selectors.put(segmentId, selector); + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); + if (inventoryValue == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping addition of segment[%s].", + server.getName(), + segmentId + ); + return; + } else { + queryableDruidServer = addServer(inventoryValue); + } + } + selector.addServerAndUpdateSegment(queryableDruidServer, segment); + // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it + runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); + } + } + + private QueryableDruidServer addServer(DruidServer server) + { + QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); + QueryableDruidServer exists = clients.put(server.getName(), retVal); + if (exists != null) { + log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); + } + + return retVal; + } + + private DirectDruidClient makeDirectClient(DruidServer server) + { + return new DirectDruidClient( + warehouse, + queryWatcher, + smileMapper, + httpClient, + server.getScheme(), + server.getHost(), + emitter + ); + } + + private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + final ServerSelector selector; + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + selector = selectors.get(segmentId); + if (selector == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping removal of segment[%s].", + server.getName(), + segmentId + ); + } else if (!selector.removeServer(queryableDruidServer)) { + log.warn( + "Asked to disassociate non-existant association between server[%s] and segment[%s]", + server, + segmentId + ); + } else { + runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } + + if (selector.isEmpty()) { + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + selectors.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } else { + runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); + } + } + } + } + + @Override + public VersionedIntervalTimeline getTimeline(DataSource dataSource) + { + String table = Iterables.getOnlyElement(dataSource.getTableNames()); + synchronized (lock) { + timelines.get(table).getAllTimelineEntries() + } + } + + @Override + public Map getSegmentLoadInfos() + { + return CollectionUtils.mapValues(selectors, ServerSelector::toSegmentLoadInfo); + } + + @Override + public DruidServer getInventoryValue(String serverKey) + { + return baseView.getInventoryValue(serverKey); + } + + @Override + public Collection getInventory() + { + return baseView.getInventory(); + } + + @Override + public boolean isStarted() + { + return baseView.isStarted(); + } + + @Override + public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) + { + return baseView.isSegmentLoadedByServer(serverKey, segment); + } + + private void runTimelineCallbacks(final Function function) + { + for (Map.Entry entry : timelineCallbacks.entrySet()) { + entry.getValue().execute( + () -> { + if (CallbackAction.UNREGISTER == function.apply(entry.getKey())) { + timelineCallbacks.remove(entry.getKey()); + } + } + ); + } + } + + @Override + public void registerTimelineCallback(Executor exec, TimelineCallback callback) + { + timelineCallbacks.put(callback, exec); + } + + @Override + public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) + { + throw new UnsupportedOperationException(); + } + + @Override + public void registerSegmentCallback(Executor exec, SegmentCallback callback) + { + baseView.registerSegmentCallback(exec, callback); + } + + @Override + public Optional> getTimeline(DataSourceAnalysis analysis) + { + final TableDataSource table = + analysis.getBaseTableDataSource() + .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); + + synchronized (lock) { + return Optional.ofNullable(timelines.get(table.getName())); + } + } + + @Override + public List getDruidServers() + { + return clients.values().stream() + .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) + .collect(Collectors.toList()); + } + + @Override + public QueryRunner getQueryRunner(DruidServer server) + { + synchronized (lock) { + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.error("No QueryRunner found for server name[%s].", server.getName()); + return null; + } + return queryableDruidServer.getQueryRunner(); + } + } +} diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java similarity index 98% rename from server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java index 1bca2a0690af..1614709fce7d 100644 --- a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java @@ -52,7 +52,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class CoordinatorServerViewTest extends CuratorTestBase +public class CoordinatorServerViewImplImplTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; @@ -63,9 +63,9 @@ public class CoordinatorServerViewTest extends CuratorTestBase private CountDownLatch segmentRemovedLatch; private BatchServerInventoryView baseView; - private CoordinatorServerView overlordServerView; + private CoordinatorServerViewImpl overlordServerView; - public CoordinatorServerViewTest() + public CoordinatorServerViewImplImplTest() { jsonMapper = TestHelper.makeJsonMapper(); zkPathsConfig = new ZkPathsConfig(); @@ -332,7 +332,7 @@ public CallbackAction segmentViewInitialized() } }; - overlordServerView = new CoordinatorServerView( + overlordServerView = new CoordinatorServerViewImpl( baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter() From a48d2fe271fff95d9f4650592548286d8f62ffa8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 28 Aug 2023 18:30:24 +0530 Subject: [PATCH 03/82] minor change --- .../java/org/apache/druid/client/CoordinatorServerViewImpl.java | 2 +- ...ViewImplImplImplTest.java => CoordinatorServerViewTest.java} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename server/src/test/java/org/apache/druid/client/{CoordinatorServerViewImplImplImplTest.java => CoordinatorServerViewTest.java} (99%) diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java index cfcab66105bd..deab4e32e897 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java @@ -28,7 +28,7 @@ @ManageLifecycle public class CoordinatorServerViewImpl implements CoordinatorServerView { - private static final Logger log = new Logger(Alpha.class); + private static final Logger log = new Logger(CoordinatorServerViewImpl.class); private final Object lock = new Object(); diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java similarity index 99% rename from server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java index 1614709fce7d..11a38af228b6 100644 --- a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java @@ -52,7 +52,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class CoordinatorServerViewImplImplTest extends CuratorTestBase +public class CoordinatorServerViewTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; From 5e7756f784b8aaeb2d514acb88481aeb7926f7c2 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 28 Aug 2023 18:37:26 +0530 Subject: [PATCH 04/82] Revert "refactor CoordinatorServerView" This reverts commit d1c0dca1494f6fdbeff74cd17170232f8c9450ea. --- .../timeline/SegmentStatusInCluster.java | 16 +- .../druid/client/CoordinatorServerView.java | 221 ++++++++++++++++- .../client/CoordinatorServerViewImpl.java | 224 ------------------ .../CoordinatorServerViewInterface.java | 14 ++ .../TimelineAwareCoordinatorServerView.java | 170 ++++++++++--- .../druid/client/selector/ServerSelector.java | 8 + .../metadata/AvailableSegmentMetadata.java | 4 +- .../segment/metadata/DatasourceSchema.java | 1 + .../metadata/SegmentMetadataCache.java | 15 +- .../server/http/DataSourcesResource.java | 20 +- .../druid/server/http/MetadataResource.java | 51 ++-- ...melineAwareCoordinatorServerViewTest.java} | 8 +- .../CuratorDruidCoordinatorTest.java | 6 +- .../server/http/DataSourcesResourceTest.java | 6 +- .../server/http/ServersResourceTest.java | 4 +- .../org/apache/druid/cli/CliCoordinator.java | 9 +- .../schema/BrokerSegmentMetadataCache.java | 21 ++ .../schema/BrokerSegmentMetadataView.java | 14 +- .../sql/calcite/schema/SystemSchema.java | 3 +- 19 files changed, 492 insertions(+), 323 deletions(-) delete mode 100644 server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java create mode 100644 server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java rename server/src/test/java/org/apache/druid/client/{CoordinatorServerViewTest.java => TimelineAwareCoordinatorServerViewTest.java} (98%) diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index 26d89d79ed34..263f1619c6f3 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -67,12 +67,12 @@ public SegmentStatusInCluster( @JsonProperty("overshadowed") boolean overshadowed, @JsonProperty("replicationFactor") @Nullable Integer replicationFactor, @JsonProperty("numRows") @Nullable Long numRows, - @JsonProperty("isRealtime") Long isRealtime, - @JsonProperty("isPublished") boolean isPublished + @JsonProperty("isPublished") boolean isPublished, + @JsonProperty("isRealtime") Long isRealtime ) { // Jackson will overwrite dataSegment if needed (even though the field is final) - this(null, overshadowed, replicationFactor, numRows, isRealtime, isPublished); + this(null, overshadowed, replicationFactor, numRows, isPublished, isRealtime); } public SegmentStatusInCluster( @@ -80,8 +80,8 @@ public SegmentStatusInCluster( boolean overshadowed, Integer replicationFactor, Long numRows, - Long isRealtime, - boolean isPublished + boolean isPublished, + Long isRealtime ) { this.dataSegment = dataSegment; @@ -144,13 +144,14 @@ public boolean equals(Object o) && Objects.equals(replicationFactor, that.replicationFactor) && Objects.equals(dataSegment, that.dataSegment) && Objects.equals(numRows, that.numRows) - && Objects.equals(isRealtime, that.isRealtime); + && Objects.equals(isRealtime, that.isRealtime) + && isPublished == that.isPublished; } @Override public int hashCode() { - return Objects.hash(overshadowed, replicationFactor, dataSegment, numRows, isRealtime); + return Objects.hash(overshadowed, replicationFactor, dataSegment, numRows, isPublished, isRealtime); } @Override @@ -167,6 +168,7 @@ public String toString() ", replicationFactor=" + replicationFactor + ", dataSegment=" + dataSegment + ", numRows=" + numRows + + ", isPublished=" + isPublished + ", isRealtime=" + isRealtime + '}'; } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index b995ed70b7fa..6b6fecc2af6b 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -1,15 +1,224 @@ package org.apache.druid.client; -import org.apache.druid.client.InventoryView; -import org.apache.druid.client.SegmentLoadInfo; +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.inject.Inject; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DataSource; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.PartitionChunk; +import java.util.Collection; +import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; -public interface CoordinatorServerView extends InventoryView +/** + * ServerView of coordinator for the state of segments being loaded in the cluster. + */ +@ManageLifecycle +public class CoordinatorServerView implements CoordinatorServerViewInterface { - VersionedIntervalTimeline getTimeline(DataSource dataSource); - Map getSegmentLoadInfos(); -} \ No newline at end of file + private static final Logger log = new Logger(CoordinatorServerView.class); + + private final Object lock = new Object(); + + private final Map segmentLoadInfos; + private final Map> timelines; + + private final ServerInventoryView baseView; + private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; + + private final CountDownLatch initialized = new CountDownLatch(1); + private final ServiceEmitter emitter; + + @Inject + public CoordinatorServerView( + ServerInventoryView baseView, + CoordinatorSegmentWatcherConfig segmentWatcherConfig, + ServiceEmitter emitter + ) + { + this.baseView = baseView; + this.segmentWatcherConfig = segmentWatcherConfig; + this.emitter = emitter; + this.segmentLoadInfos = new HashMap<>(); + this.timelines = new HashMap<>(); + + ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); + baseView.registerSegmentCallback( + exec, + new ServerView.SegmentCallback() + { + @Override + public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + serverAddedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) + { + serverRemovedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentViewInitialized() + { + initialized.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + + baseView.registerServerRemovedCallback( + exec, + new ServerView.ServerRemovedCallback() + { + @Override + public ServerView.CallbackAction serverRemoved(DruidServer server) + { + removeServer(server); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + } + + @LifecycleStart + public void start() throws InterruptedException + { + if (segmentWatcherConfig.isAwaitInitializationOnStart()) { + final long startMillis = System.currentTimeMillis(); + log.info("%s waiting for initialization.", getClass().getSimpleName()); + initialized.await(); + final long endMillis = System.currentTimeMillis(); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); + emitter.emit(ServiceMetricEvent.builder().build( + "serverview/init/time", + endMillis - startMillis + )); + } + } + + private void removeServer(DruidServer server) + { + for (DataSegment segment : server.iterateAllSegments()) { + serverRemovedSegment(server.getMetadata(), segment); + } + } + + private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) + { + SegmentId segmentId = segment.getId(); + synchronized (lock) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + + SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + // servers escape the scope of this object so use ConcurrentSet + segmentLoadInfo = new SegmentLoadInfo(segment); + + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + if (timeline == null) { + timeline = new VersionedIntervalTimeline<>(Ordering.natural()); + timelines.put(segment.getDataSource(), timeline); + } + + timeline.add( + segment.getInterval(), + segment.getVersion(), + segment.getShardSpec().createChunk(segmentLoadInfo) + ); + segmentLoadInfos.put(segmentId, segmentLoadInfo); + } + segmentLoadInfo.addServer(server); + } + } + + private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) + { + SegmentId segmentId = segment.getId(); + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + segmentLoadInfo.removeServer(server); + if (segmentLoadInfo.isEmpty()) { + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + segmentLoadInfos.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( + new SegmentLoadInfo( + segment + ) + ) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } + } + } + } + + @Override + public VersionedIntervalTimeline getTimeline(DataSource dataSource) + { + String table = Iterables.getOnlyElement(dataSource.getTableNames()); + synchronized (lock) { + return timelines.get(table); + } + } + + @Override + public Map getSegmentLoadInfos() + { + return segmentLoadInfos; + } + + @Override + public DruidServer getInventoryValue(String serverKey) + { + return baseView.getInventoryValue(serverKey); + } + + @Override + public Collection getInventory() + { + return baseView.getInventory(); + } + + @Override + public boolean isStarted() + { + return baseView.isStarted(); + } + + @Override + public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) + { + return baseView.isSegmentLoadedByServer(serverKey, segment); + } +} diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java deleted file mode 100644 index deab4e32e897..000000000000 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java +++ /dev/null @@ -1,224 +0,0 @@ -package org.apache.druid.client; - -import com.google.common.collect.Iterables; -import com.google.common.collect.Ordering; -import com.google.inject.Inject; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.apache.druid.query.DataSource; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.PartitionChunk; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; - -/** - * ServerView of coordinator for the state of segments being loaded in the cluster. - */ -@ManageLifecycle -public class CoordinatorServerViewImpl implements CoordinatorServerView -{ - private static final Logger log = new Logger(CoordinatorServerViewImpl.class); - - private final Object lock = new Object(); - - private final Map segmentLoadInfos; - private final Map> timelines; - - private final ServerInventoryView baseView; - private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; - - private final CountDownLatch initialized = new CountDownLatch(1); - private final ServiceEmitter emitter; - - @Inject - public CoordinatorServerViewImpl( - ServerInventoryView baseView, - CoordinatorSegmentWatcherConfig segmentWatcherConfig, - ServiceEmitter emitter - ) - { - this.baseView = baseView; - this.segmentWatcherConfig = segmentWatcherConfig; - this.emitter = emitter; - this.segmentLoadInfos = new HashMap<>(); - this.timelines = new HashMap<>(); - - ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); - baseView.registerSegmentCallback( - exec, - new ServerView.SegmentCallback() - { - @Override - public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) - { - serverAddedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) - { - serverRemovedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentViewInitialized() - { - initialized.countDown(); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - - baseView.registerServerRemovedCallback( - exec, - new ServerView.ServerRemovedCallback() - { - @Override - public ServerView.CallbackAction serverRemoved(DruidServer server) - { - removeServer(server); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - } - - @LifecycleStart - public void start() throws InterruptedException - { - if (segmentWatcherConfig.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); - initialized.await(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( - "serverview/init/time", - endMillis - startMillis - )); - } - } - - private void removeServer(DruidServer server) - { - for (DataSegment segment : server.iterateAllSegments()) { - serverRemovedSegment(server.getMetadata(), segment); - } - } - - private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) - { - SegmentId segmentId = segment.getId(); - synchronized (lock) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - - SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - // servers escape the scope of this object so use ConcurrentSet - segmentLoadInfo = new SegmentLoadInfo(segment); - - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - if (timeline == null) { - timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - timelines.put(segment.getDataSource(), timeline); - } - - timeline.add( - segment.getInterval(), - segment.getVersion(), - segment.getShardSpec().createChunk(segmentLoadInfo) - ); - segmentLoadInfos.put(segmentId, segmentLoadInfo); - } - segmentLoadInfo.addServer(server); - } - } - - private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) - { - SegmentId segmentId = segment.getId(); - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - segmentLoadInfo.removeServer(server); - if (segmentLoadInfo.isEmpty()) { - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - segmentLoadInfos.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( - new SegmentLoadInfo( - segment - ) - ) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } - } - } - } - - @Override - public VersionedIntervalTimeline getTimeline(DataSource dataSource) - { - String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (lock) { - return timelines.get(table); - } - } - - @Override - public Map getSegmentLoadInfos() - { - return segmentLoadInfos; - } - - @Override - public DruidServer getInventoryValue(String serverKey) - { - return baseView.getInventoryValue(serverKey); - } - - @Override - public Collection getInventory() - { - return baseView.getInventory(); - } - - @Override - public boolean isStarted() - { - return baseView.isStarted(); - } - - @Override - public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) - { - return baseView.isSegmentLoadedByServer(serverKey, segment); - } -} diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java new file mode 100644 index 000000000000..4bbfdf392892 --- /dev/null +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java @@ -0,0 +1,14 @@ +package org.apache.druid.client; + +import org.apache.druid.query.DataSource; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.VersionedIntervalTimeline; + +import java.util.Map; + +public interface CoordinatorServerViewInterface extends InventoryView +{ + + VersionedIntervalTimeline getTimeline(DataSource dataSource); + Map getSegmentLoadInfos(); +} diff --git a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java index a58c2e18957f..a4edb91a3e4c 100644 --- a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.client; import com.fasterxml.jackson.databind.ObjectMapper; @@ -24,18 +43,19 @@ import org.apache.druid.query.TableDataSource; import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.TimelineLookup; import org.apache.druid.timeline.VersionedIntervalTimeline; import org.apache.druid.timeline.partition.PartitionChunk; -import org.apache.druid.utils.CollectionUtils; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @@ -44,16 +64,21 @@ import java.util.function.Function; import java.util.stream.Collectors; +/** + * ServerView of coordinator for the state of segments being loaded in the cluster. + */ @ManageLifecycle -public class TimelineAwareCoordinatorServerView implements CoordinatorServerView, TimelineServerView +public class TimelineAwareCoordinatorServerView implements CoordinatorServerViewInterface, TimelineServerView { private static final Logger log = new Logger(TimelineAwareCoordinatorServerView.class); private final Object lock = new Object(); + private final Map segmentLoadInfos; private final ConcurrentMap clients = new ConcurrentHashMap<>(); private final Map selectors = new HashMap<>(); - private final Map> timelines = new HashMap<>(); + private final Map> timelines2 = new HashMap<>(); + private final Map> timelines; private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); @@ -85,6 +110,8 @@ public TimelineAwareCoordinatorServerView( this.baseView = baseView; this.segmentWatcherConfig = segmentWatcherConfig; this.emitter = emitter; + this.segmentLoadInfos = new HashMap<>(); + this.timelines = new HashMap<>(); this.tierSelectorStrategy = tierSelectorStrategy; this.warehouse = warehouse; this.queryWatcher = queryWatcher; @@ -106,6 +133,7 @@ public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSe @Override public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) { + serverRemovedSegment2(server, segment); serverRemovedSegment(server, segment); return ServerView.CallbackAction.CONTINUE; } @@ -153,45 +181,80 @@ public void start() throws InterruptedException private void removeServer(DruidServer server) { for (DataSegment segment : server.iterateAllSegments()) { + serverAddedSegment2(server.getMetadata(), segment); serverRemovedSegment(server.getMetadata(), segment); } } private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) { - final SegmentId segmentId = segment.getId(); + SegmentId segmentId = segment.getId(); synchronized (lock) { log.debug("Adding segment[%s] for server[%s]", segment, server); - ServerSelector selector = selectors.get(segmentId); - if (selector == null) { - selector = new ServerSelector(segment, tierSelectorStrategy); - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + // servers escape the scope of this object so use ConcurrentSet + segmentLoadInfo = new SegmentLoadInfo(segment); + + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); if (timeline == null) { - // broker needs to skip tombstones timeline = new VersionedIntervalTimeline<>(Ordering.natural()); timelines.put(segment.getDataSource(), timeline); } - timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); - selectors.put(segmentId, selector); + timeline.add( + segment.getInterval(), + segment.getVersion(), + segment.getShardSpec().createChunk(segmentLoadInfo) + ); + segmentLoadInfos.put(segmentId, segmentLoadInfo); } + segmentLoadInfo.addServer(server); + } + runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); + } - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); - if (inventoryValue == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping addition of segment[%s].", - server.getName(), - segmentId - ); - return; - } else { - queryableDruidServer = addServer(inventoryValue); + private void serverAddedSegment2(final DruidServerMetadata server, final DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + synchronized (lock) { + // in theory we could probably just filter this to ensure we don't put ourselves in here, to make broker tree + // query topologies, but for now just skip all brokers, so we don't create some sort of wild infinite query + // loop... + if (!server.getType().equals(ServerType.BROKER)) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + ServerSelector selector = selectors.get(segmentId); + if (selector == null) { + selector = new ServerSelector(segment, tierSelectorStrategy); + + VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); + if (timeline == null) { + // broker needs to skip tombstones + timeline = new VersionedIntervalTimeline<>(Ordering.natural(), true); + timelines2.put(segment.getDataSource(), timeline); + } + + timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); + selectors.put(segmentId, selector); + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); + if (inventoryValue == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping addition of segment[%s].", + server.getName(), + segmentId + ); + return; + } else { + queryableDruidServer = addServer(inventoryValue); + } } + selector.addServerAndUpdateSegment(queryableDruidServer, segment); } - selector.addServerAndUpdateSegment(queryableDruidServer, segment); // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); } @@ -222,6 +285,46 @@ private DirectDruidClient makeDirectClient(DruidServer server) } private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) + { + SegmentId segmentId = segment.getId(); + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + if (segmentLoadInfo.removeServer(server)) { + runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } + if (segmentLoadInfo.isEmpty()) { + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + segmentLoadInfos.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( + new SegmentLoadInfo( + segment + ) + ) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } else { + runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); + } + } + } + } + + private void serverRemovedSegment2(DruidServerMetadata server, DataSegment segment) { final SegmentId segmentId = segment.getId(); final ServerSelector selector; @@ -229,6 +332,8 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen synchronized (lock) { log.debug("Removing segment[%s] from server[%s].", segmentId, server); + // we don't store broker segments here, but still run the callbacks for the segment being removed from the server + // since the broker segments are not stored on the timeline, do not fire segmentRemoved event selector = selectors.get(segmentId); if (selector == null) { log.warn("Told to remove non-existant segment[%s]", segmentId); @@ -249,11 +354,11 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen segmentId ); } else { - runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + // runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); } if (selector.isEmpty()) { - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); selectors.remove(segmentId); final PartitionChunk removedPartition = timeline.remove( @@ -267,7 +372,7 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen segment.getVersion() ); } else { - runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); + // runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); } } } @@ -278,14 +383,19 @@ public VersionedIntervalTimeline getTimeline(DataSource { String table = Iterables.getOnlyElement(dataSource.getTableNames()); synchronized (lock) { - timelines.get(table).getAllTimelineEntries() + // build a new timeline? } } @Override public Map getSegmentLoadInfos() { - return CollectionUtils.mapValues(selectors, ServerSelector::toSegmentLoadInfo); + // map + } + + public Set getLoadedSegmentIds() + { + return segmentLoadInfos.keySet(); } @Override @@ -351,7 +461,7 @@ public Optional> getTimeline(Da .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); synchronized (lock) { - return Optional.ofNullable(timelines.get(table.getName())); + return Optional.ofNullable(timelines2.get(table.getName())); } } diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 583681bf7674..8bc97a7b6efa 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -19,8 +19,11 @@ package org.apache.druid.client.selector; +import com.google.common.collect.Sets; import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap; import org.apache.druid.client.DataSegmentInterner; +import org.apache.druid.client.ImmutableSegmentLoadInfo; +import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunner; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -216,4 +219,9 @@ public boolean hasData() return segment.get().hasData(); } + public SegmentLoadInfo toSegmentLoadInfo() + { + List allServers = getAllServers(); + return new SegmentLoadInfo(segment.get(), Sets.newConcurrentHashSet(allServers)); + } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java index 104194531769..cd2f849438ee 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java @@ -163,11 +163,9 @@ public AvailableSegmentMetadata build() public String toString() { return "AvailableSegmentMetadata{" + - "segment=" + segment + + "segmentId=" + segment.getId() + ", isRealtime=" + isRealtime + - ", segmentServers=" + segmentServers + ", numRows=" + numRows + - ", rowSignature=" + rowSignature + '}'; } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java index cad5cb882c62..9fb13fe5f0e5 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java @@ -6,6 +6,7 @@ public class DatasourceSchema { + // dsinfo private final String datasource; private final RowSignature rowSignature; diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 971c45cd3885..850f9746e7f2 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -281,6 +281,17 @@ public ServerView.CallbackAction serverSegmentRemoved( ); } + public void removeFromTable(String s) + { + tables.remove(s); + } + + public boolean tablesContains(String s) + { + return tables.containsKey(s); + } + + private void startCacheExec() { cacheExec.submit( @@ -523,7 +534,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } ); } - if (!tables.containsKey(segment.getDataSource())) { + if (!tablesContains(segment.getDataSource())) { refreshImmediately = true; } @@ -554,7 +565,7 @@ void removeSegment(final DataSegment segment) totalSegments--; } if (segmentsMap.isEmpty()) { - tables.remove(segment.getDataSource()); + removeFromTable(segment.getDataSource()); log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); return null; } else { diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 9fa73970100f..70ad14f625e6 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,12 +30,12 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableSegmentLoadInfo; -import org.apache.druid.client.SegmentLoadInfo; +import org.apache.druid.client.selector.ServerSelector; import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.DateTimes; @@ -102,7 +102,7 @@ public class DataSourcesResource private static final Logger log = new Logger(DataSourcesResource.class); private static final long DEFAULT_LOADSTATUS_INTERVAL_OFFSET = 14 * 24 * 60 * 60 * 1000; - private final CoordinatorServerView serverInventoryView; + private final Alpha serverInventoryView; private final SegmentsMetadataManager segmentsMetadataManager; private final MetadataRuleManager metadataRuleManager; private final OverlordClient overlordClient; @@ -111,7 +111,7 @@ public class DataSourcesResource @Inject public DataSourcesResource( - CoordinatorServerView serverInventoryView, + Alpha serverInventoryView, SegmentsMetadataManager segmentsMetadataManager, MetadataRuleManager metadataRuleManager, @Nullable OverlordClient overlordClient, @@ -820,7 +820,7 @@ public Response getServedSegmentsInInterval( @QueryParam("partial") final boolean partial ) { - TimelineLookup timeline = serverInventoryView.getTimeline( + TimelineLookup timeline = serverInventoryView.getTimeline( new TableDataSource(dataSourceName) ); final Interval theInterval = Intervals.of(interval.replace('_', '/')); @@ -833,19 +833,19 @@ public Response getServedSegmentsInInterval( } private Iterable prepareServedSegmentsInInterval( - TimelineLookup dataSourceServingTimeline, + TimelineLookup dataSourceServingTimeline, Interval interval ) { - Iterable> lookup = + Iterable> lookup = dataSourceServingTimeline.lookupWithIncompletePartitions(interval); return FunctionalIterable .create(lookup) .transformCat( - (TimelineObjectHolder input) -> + (TimelineObjectHolder input) -> Iterables.transform( input.getObject(), - (PartitionChunk chunk) -> chunk.getObject().toImmutableSegmentLoadInfo() + (PartitionChunk chunk) -> chunk.getObject().toImmutableSegmentLoadInfo() ) ); } @@ -885,7 +885,7 @@ public Response isHandOffComplete( return Response.ok(true).build(); } - TimelineLookup timeline = serverInventoryView.getTimeline( + TimelineLookup timeline = serverInventoryView.getTimeline( new TableDataSource(dataSourceName) ); if (timeline == null) { diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 9c9622e05e50..3f3060be6683 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -28,6 +28,7 @@ import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.Segments; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.segment.metadata.SegmentMetadataCache; @@ -42,6 +43,7 @@ import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.joda.time.Interval; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; @@ -70,6 +72,7 @@ @Path("/druid/coordinator/v1/metadata") public class MetadataResource { + private final Logger log = new Logger(MetadataResource.class); private final SegmentsMetadataManager segmentsMetadataManager; private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; @@ -145,7 +148,8 @@ public Response getDataSources( public Response getAllUsedSegments( @Context final HttpServletRequest req, @QueryParam("datasources") final @Nullable Set dataSources, - @QueryParam("includeOvershadowedStatus") final @Nullable String includeOvershadowedStatus + @QueryParam("includeOvershadowedStatus") final @Nullable String includeOvershadowedStatus, + @QueryParam("includeOvershadowedStatus") final @Nullable String includeRealtime ) { if (includeOvershadowedStatus != null) { @@ -188,45 +192,52 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .filter(dataSourceWithUsedSegments -> dataSources.contains(dataSourceWithUsedSegments.getName())) .collect(Collectors.toList()); } - final Stream usedSegments = dataSourcesWithUsedSegments - .stream() - .flatMap(t -> t.getSegments().stream()); final Set overshadowedSegments = dataSourcesSnapshot.getOvershadowedSegments(); + final Set segmentAlreadySeen = new HashSet<>(); + final Stream segmentStatus = dataSourcesWithUsedSegments + .stream() + .flatMap(t -> t.getSegments().stream()) + .map(segment -> { + // The replication factor for unloaded segments is 0 as they will be unloaded soon + boolean isOvershadowed = overshadowedSegments.contains(segment); + AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( + segment.getDataSource(), + segment.getId() + ); + Integer replicationFactor = isOvershadowed ? (Integer) 0 + : coordinator.getReplicationFactor(segment.getId()); + Long numRows = (null != availableSegmentMetadata) ? availableSegmentMetadata.getNumRows() : null; + segmentAlreadySeen.add(segment.getId()); + return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, 20L, true, 0L); + }).peek(v -> log.info("peeking into first stream segmentseen [%s], id [%s] isPublished [%s]", segmentAlreadySeen, v.getDataSegment().getId(), v.isPublished())); - Set segmentAlreadySeen = new HashSet<>(); - final Stream segmentStatus = usedSegments.map(segment -> { - // The replication factor for unloaded segments is 0 as they will be unloaded soon - boolean isOvershadowed = overshadowedSegments.contains(segment); - AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata(segment.getDataSource(), segment.getId()); - Integer replicationFactor = isOvershadowed ? (Integer) 0 - : coordinator.getReplicationFactor(segment.getId()); - Long numRows = (null != availableSegmentMetadata) ? availableSegmentMetadata.getNumRows() : null; - segmentAlreadySeen.add(segment.getId()); - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, numRows, 0L, true); - }); + log.info("printing the content in smc cache %s", segmentMetadataCache.getSegmentMetadataSnapshot().values()); final Stream realtimeSegmentStatus = segmentMetadataCache .getSegmentMetadataSnapshot() .values() .stream() + .peek(v -> log.info("peeking into second stream (first) segmentseen [%s], id [%s]", segmentAlreadySeen, v.getSegment().getId())) .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment().getId())) .map(availableSegmentMetadata -> { return new SegmentStatusInCluster( availableSegmentMetadata.getSegment(), false, (int) availableSegmentMetadata.getNumReplicas(), - availableSegmentMetadata.getNumRows(), - availableSegmentMetadata.isRealtime(), - false + /**availableSegmentMetadata.getNumRows(), **/ 30L, + false, + availableSegmentMetadata.isRealtime() ); - }); + }).peek(v -> log.info("peeking into second stream (second) segmentseen [%s], id [%s]", segmentAlreadySeen, v.getDataSegment().getId()));; + + Stream combined = Stream.concat(segmentStatus, realtimeSegmentStatus).peek(v -> log.info("combined stream element %s", v)); final Function> raGenerator = segment -> Collections .singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSegment().getDataSource())); final Iterable authorizedSegments = AuthorizationUtils.filterAuthorizedResources( req, - Stream.concat(segmentStatus, realtimeSegmentStatus)::iterator, + combined::iterator, raGenerator, authorizerMapper ); diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java similarity index 98% rename from server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java rename to server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java index 11a38af228b6..190a41e53f9e 100644 --- a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java @@ -52,7 +52,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class CoordinatorServerViewTest extends CuratorTestBase +public class TimelineAwareCoordinatorServerViewTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; @@ -63,9 +63,9 @@ public class CoordinatorServerViewTest extends CuratorTestBase private CountDownLatch segmentRemovedLatch; private BatchServerInventoryView baseView; - private CoordinatorServerViewImpl overlordServerView; + private TimelineAwareCoordinatorServerView overlordServerView; - public CoordinatorServerViewImplImplTest() + public TimelineAwareCoordinatorServerViewTest() { jsonMapper = TestHelper.makeJsonMapper(); zkPathsConfig = new ZkPathsConfig(); @@ -332,7 +332,7 @@ public CallbackAction segmentViewInitialized() } }; - overlordServerView = new CoordinatorServerViewImpl( + overlordServerView = new TimelineAwareCoordinatorServerView( baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter() diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index b9b6fc8516fe..647eab9c6c45 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -30,7 +30,7 @@ import org.apache.curator.utils.ZKPaths; import org.apache.druid.client.BatchServerInventoryView; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -97,7 +97,7 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase private static final long COORDINATOR_PERIOD = 100; private BatchServerInventoryView baseView; - private CoordinatorServerView serverView; + private Alpha serverView; private CountDownLatch segmentViewInitLatch; /** * The following two fields are changed during {@link #testMoveSegment()}, the change might not be visible from the @@ -405,7 +405,7 @@ public CallbackAction segmentViewInitialized() } }; - serverView = new CoordinatorServerView(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); + serverView = new Alpha(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); baseView.start(); diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 94b9bf8a84d0..5cdf1d5b73b9 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -28,7 +28,7 @@ import com.google.common.util.concurrent.Futures; import it.unimi.dsi.fastutil.objects.Object2LongMap; import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -83,7 +83,7 @@ public class DataSourcesResourceTest { - private CoordinatorServerView inventoryView; + private Alpha inventoryView; private DruidServer server; private List listDataSources; private List dataSegmentList; @@ -94,7 +94,7 @@ public class DataSourcesResourceTest public void setUp() { request = EasyMock.createStrictMock(HttpServletRequest.class); - inventoryView = EasyMock.createStrictMock(CoordinatorServerView.class); + inventoryView = EasyMock.createStrictMock(Alpha.class); server = EasyMock.niceMock(DruidServer.class); dataSegmentList = new ArrayList<>(); dataSegmentList.add( diff --git a/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java b/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java index f96d2a3e74c8..951070557d7d 100644 --- a/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.DruidServer; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; @@ -53,7 +53,7 @@ public void setUp() .build(); dummyServer.addDataSegment(segment); - CoordinatorServerView inventoryView = EasyMock.createMock(CoordinatorServerView.class); + Alpha inventoryView = EasyMock.createMock(Alpha.class); EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(dummyServer)).anyTimes(); EasyMock.expect(inventoryView.getInventoryValue(dummyServer.getName())).andReturn(dummyServer).anyTimes(); EasyMock.replay(inventoryView); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 93e7dad644e4..75475eac6f9b 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -40,7 +40,7 @@ import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.HttpServerInventoryViewResource; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; @@ -52,7 +52,6 @@ import org.apache.druid.guice.BrokerProcessingModule; import org.apache.druid.guice.ConditionalMultibind; import org.apache.druid.guice.ConfigProvider; -import org.apache.druid.guice.DruidProcessingModule; import org.apache.druid.guice.Jerseys; import org.apache.druid.guice.JoinableFactoryModule; import org.apache.druid.guice.JsonConfigProvider; @@ -257,11 +256,11 @@ public void configure(Binder binder) binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class); binder.bind(CachingClusteredClient.class).in(LazySingleton.class); - binder.bind(CoordinatorServerView.class); - binder.bind(TimelineServerView.class).to(CoordinatorServerView.class).in(LazySingleton.class); + binder.bind(Alpha.class); + binder.bind(TimelineServerView.class).to(Alpha.class).in(LazySingleton.class); binder.bind(DruidCoordinator.class); - LifecycleModule.register(binder, CoordinatorServerView.class); + LifecycleModule.register(binder, Alpha.class); LifecycleModule.register(binder, MetadataStorage.class); LifecycleModule.register(binder, DruidCoordinator.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 0224841a6cbc..09f252cc3e20 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -1,5 +1,6 @@ package org.apache.druid.sql.calcite.schema; +import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; @@ -13,6 +14,8 @@ import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -68,6 +71,24 @@ public void rebuildDatasource(String dataSource) } } + @Override + public Set getDatasourceNames() + { + return tables.keySet(); + } + + @Override + public void removeFromTable(String s) + { + tables.remove(s); + } + + @Override + public boolean tablesContains(String s) + { + return tables.containsKey(s); + } + public DatasourceTable.PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) { return tables.get(name); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index cf92bba0eb69..8b737ba9eb48 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -60,9 +60,13 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; + import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + + + /** * This class polls the Coordinator in background to keep the latest published segments. * Provides {@link #getSegmentMetadata()} for others to get segments in metadata store. @@ -204,8 +208,8 @@ private ImmutableSortedSet fetchSegmentMetadata() segment.isOvershadowed(), replicationFactor, segment.getNumRows(), - segment.isRealtime(), - segment.isPublished() + true, + segment.isRealtime() ); builder.add(segmentStatusInCluster); } @@ -219,7 +223,8 @@ private void pollDatasourceSchema() Map physicalDatasourceMetadataMap = new HashMap<>(); - for (List partition : Iterables.partition(datasources, 10)) { + for (List partition : Iterables.partition(datasources, 100)) { + // retain watched datasources List datasourceSchemas = FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema( partition), true); @@ -246,6 +251,7 @@ ImmutableSortedSet getSegmentMetadata() protected SystemSchema.SegmentsTable.SegmentTableView getSegmentTableView() { ImmutableSortedSet allSegmentMetadata = getSegmentMetadata(); + log.info("logging polled segments from coordinator %s", allSegmentMetadata); final ImmutableSortedSet.Builder publishedSegmentBuilder = ImmutableSortedSet.naturalOrder(); Map availableSegmentMetadataMap; @@ -262,12 +268,14 @@ protected SystemSchema.SegmentsTable.SegmentTableView getSegmentTableView() { // build available segment metadata map by combining stuff from brokerServerView and numRows from data returned from coordinator availableSegmentMetadataMap = new HashMap<>(); Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); + // only look at watched ds, confirm if brokerServerView is also looking for watched ds for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { if (segmentStatusInCluster.isPublished()) { publishedSegmentBuilder.add(segmentStatusInCluster); } SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); if (!brokerSegmentMetadata.containsKey(segmentId)) { + // log and count ignored segments continue; } ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index f1063c521ef0..73610534493a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -444,8 +444,9 @@ private Iterator> getAuthorizedAvaila protected static class SegmentTableView { + // pass the stitched private final Map availableSegmentMetadata; - private final Iterator publishedSegments; + private final Iterator publishedSegments; // coordinator private final int totalSegmentsCount; From a9ff64075fc81990390c361d0e0d678f292bd062 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 4 Sep 2023 13:55:50 +0530 Subject: [PATCH 05/82] Draft changes for the coordinator to conditionally build smc, and refactor on broker side --- .../apache/druid/client/BrokerServerView.java | 20 + ...ace.java => CoordinatorInventoryView.java} | 3 +- .../druid/client/CoordinatorServerView.java | 2 +- .../TimelineAwareCoordinatorServerView.java | 415 +----------------- .../client/coordinator/CoordinatorClient.java | 2 +- .../coordinator/CoordinatorClientImpl.java | 12 +- .../druid/client/selector/ServerSelector.java | 10 +- .../metadata/SegmentsMetadataManager.java | 5 - .../metadata/SegmentMetadataCacheConfig.java | 2 - .../server/http/DataSourcesResource.java | 24 +- .../org/apache/druid/cli/CliCoordinator.java | 63 ++- .../BrokerSegmentMetadataCacheConfig.java | 8 +- .../schema/BrokerSegmentMetadataView.java | 290 +++++++----- .../schema/DruidCalciteSchemaModule.java | 10 +- .../sql/calcite/schema/SystemSchema.java | 187 ++++---- 15 files changed, 388 insertions(+), 665 deletions(-) rename server/src/main/java/org/apache/druid/client/{CoordinatorServerViewInterface.java => CoordinatorInventoryView.java} (83%) diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index ee8490e1edf3..11624aad752a 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -440,4 +440,24 @@ public Set getDatasourceNames() { return timelines.keySet(); } + + Object getLock() + { + return lock; + } + + Map> getTimelines() + { + return timelines; + } + + Map getSelectors() + { + return selectors; + } + + ConcurrentMap getClients() + { + return clients; + } } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java b/server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java similarity index 83% rename from server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java rename to server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java index 4bbfdf392892..a938c8864f1b 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java @@ -6,9 +6,8 @@ import java.util.Map; -public interface CoordinatorServerViewInterface extends InventoryView +public interface CoordinatorInventoryView extends InventoryView { - VersionedIntervalTimeline getTimeline(DataSource dataSource); Map getSegmentLoadInfos(); } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index 6b6fecc2af6b..be4e90d9e600 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -26,7 +26,7 @@ * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class CoordinatorServerView implements CoordinatorServerViewInterface +public class CoordinatorServerView implements CoordinatorInventoryView { private static final Logger log = new Logger(CoordinatorServerView.class); diff --git a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java index a4edb91a3e4c..88661386f0bc 100644 --- a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java @@ -21,79 +21,33 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; -import com.google.common.collect.Ordering; import com.google.inject.Inject; -import org.apache.druid.client.selector.QueryableDruidServer; import org.apache.druid.client.selector.ServerSelector; import org.apache.druid.client.selector.TierSelectorStrategy; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.guice.annotations.EscalatedClient; import org.apache.druid.guice.annotations.Smile; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.query.DataSource; -import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.planning.DataSourceAnalysis; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.TimelineLookup; import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.PartitionChunk; +import org.apache.druid.utils.CollectionUtils; import java.util.Collection; -import java.util.HashMap; -import java.util.List; +import java.util.Comparator; import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.function.Function; -import java.util.stream.Collectors; /** * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class TimelineAwareCoordinatorServerView implements CoordinatorServerViewInterface, TimelineServerView +public class TimelineAwareCoordinatorServerView extends BrokerServerView implements CoordinatorInventoryView { - private static final Logger log = new Logger(TimelineAwareCoordinatorServerView.class); - - private final Object lock = new Object(); - - private final Map segmentLoadInfos; - private final ConcurrentMap clients = new ConcurrentHashMap<>(); - private final Map selectors = new HashMap<>(); - private final Map> timelines2 = new HashMap<>(); - private final Map> timelines; - - private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); - - private final ServerInventoryView baseView; - private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; - - private final CountDownLatch initialized = new CountDownLatch(1); - private final ServiceEmitter emitter; - - private final TierSelectorStrategy tierSelectorStrategy; - - private final QueryToolChestWarehouse warehouse; - private final QueryWatcher queryWatcher; - private final ObjectMapper smileMapper; - private final HttpClient httpClient; + private final FilteredServerInventoryView baseView; @Inject public TimelineAwareCoordinatorServerView( @@ -101,301 +55,42 @@ public TimelineAwareCoordinatorServerView( final QueryWatcher queryWatcher, final @Smile ObjectMapper smileMapper, final @EscalatedClient HttpClient httpClient, - ServerInventoryView baseView, - CoordinatorSegmentWatcherConfig segmentWatcherConfig, + FilteredServerInventoryView baseView, + TierSelectorStrategy tierSelectorStrategy, ServiceEmitter emitter, - TierSelectorStrategy tierSelectorStrategy + CoordinatorSegmentWatcherConfig segmentWatcherConfig ) { - this.baseView = baseView; - this.segmentWatcherConfig = segmentWatcherConfig; - this.emitter = emitter; - this.segmentLoadInfos = new HashMap<>(); - this.timelines = new HashMap<>(); - this.tierSelectorStrategy = tierSelectorStrategy; - this.warehouse = warehouse; - this.queryWatcher = queryWatcher; - this.smileMapper = smileMapper; - this.httpClient = httpClient; - - ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); - baseView.registerSegmentCallback( - exec, - new ServerView.SegmentCallback() - { - @Override - public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) - { - serverAddedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) - { - serverRemovedSegment2(server, segment); - serverRemovedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentViewInitialized() - { - initialized.countDown(); - runTimelineCallbacks(TimelineCallback::timelineInitialized); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - - baseView.registerServerRemovedCallback( - exec, - new ServerView.ServerRemovedCallback() - { - @Override - public ServerView.CallbackAction serverRemoved(DruidServer server) - { - removeServer(server); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - } - - @LifecycleStart - public void start() throws InterruptedException - { - if (segmentWatcherConfig.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); - initialized.await(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( - "serverview/init/time", - endMillis - startMillis - )); - } - } - - private void removeServer(DruidServer server) - { - for (DataSegment segment : server.iterateAllSegments()) { - serverAddedSegment2(server.getMetadata(), segment); - serverRemovedSegment(server.getMetadata(), segment); - } - } - - private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) - { - SegmentId segmentId = segment.getId(); - synchronized (lock) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - - SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - // servers escape the scope of this object so use ConcurrentSet - segmentLoadInfo = new SegmentLoadInfo(segment); - - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - if (timeline == null) { - timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - timelines.put(segment.getDataSource(), timeline); - } - - timeline.add( - segment.getInterval(), - segment.getVersion(), - segment.getShardSpec().createChunk(segmentLoadInfo) - ); - segmentLoadInfos.put(segmentId, segmentLoadInfo); - } - segmentLoadInfo.addServer(server); - } - runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); - } - - private void serverAddedSegment2(final DruidServerMetadata server, final DataSegment segment) - { - final SegmentId segmentId = segment.getId(); - synchronized (lock) { - // in theory we could probably just filter this to ensure we don't put ourselves in here, to make broker tree - // query topologies, but for now just skip all brokers, so we don't create some sort of wild infinite query - // loop... - if (!server.getType().equals(ServerType.BROKER)) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - ServerSelector selector = selectors.get(segmentId); - if (selector == null) { - selector = new ServerSelector(segment, tierSelectorStrategy); - - VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); - if (timeline == null) { - // broker needs to skip tombstones - timeline = new VersionedIntervalTimeline<>(Ordering.natural(), true); - timelines2.put(segment.getDataSource(), timeline); - } - - timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); - selectors.put(segmentId, selector); - } - - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); - if (inventoryValue == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping addition of segment[%s].", - server.getName(), - segmentId - ); - return; - } else { - queryableDruidServer = addServer(inventoryValue); - } - } - selector.addServerAndUpdateSegment(queryableDruidServer, segment); - } - // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it - runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); - } - } - - private QueryableDruidServer addServer(DruidServer server) - { - QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); - QueryableDruidServer exists = clients.put(server.getName(), retVal); - if (exists != null) { - log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); - } - - return retVal; - } - - private DirectDruidClient makeDirectClient(DruidServer server) - { - return new DirectDruidClient( - warehouse, - queryWatcher, - smileMapper, - httpClient, - server.getScheme(), - server.getHost(), - emitter - ); - } - - private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) - { - SegmentId segmentId = segment.getId(); - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - if (segmentLoadInfo.removeServer(server)) { - runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); - } - if (segmentLoadInfo.isEmpty()) { - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - segmentLoadInfos.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( - new SegmentLoadInfo( - segment - ) - ) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } else { - runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); - } - } - } - } - - private void serverRemovedSegment2(DruidServerMetadata server, DataSegment segment) - { - final SegmentId segmentId = segment.getId(); - final ServerSelector selector; - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - // we don't store broker segments here, but still run the callbacks for the segment being removed from the server - // since the broker segments are not stored on the timeline, do not fire segmentRemoved event - selector = selectors.get(segmentId); - if (selector == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping removal of segment[%s].", - server.getName(), - segmentId - ); - } else if (!selector.removeServer(queryableDruidServer)) { - log.warn( - "Asked to disassociate non-existant association between server[%s] and segment[%s]", - server, - segmentId - ); - } else { - // runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); - } - - if (selector.isEmpty()) { - VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); - selectors.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } else { - // runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); - } + super(warehouse, queryWatcher, smileMapper, httpClient, baseView, tierSelectorStrategy, emitter, new BrokerSegmentWatcherConfig() { + @Override + public boolean isAwaitInitializationOnStart() + { + return segmentWatcherConfig.isAwaitInitializationOnStart(); } - } + }); + this.baseView = baseView; } @Override public VersionedIntervalTimeline getTimeline(DataSource dataSource) { String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (lock) { + synchronized (getLock()) { // build a new timeline? + VersionedIntervalTimeline timeline = getTimelines().get(table); + Collection x = timeline.iterateAllObjects(); + VersionedIntervalTimeline newTimeline = new VersionedIntervalTimeline<>(Comparator.naturalOrder()); + newTimeline.addAll(x.stream().map(v -> new VersionedIntervalTimeline.PartitionChunkEntry( + v.getSegment().getInterval(), v.getSegment().getVersion(), v.getSegment().getShardSpec().createChunk(v.toSegmentLoadInfo()))).iterator()); + + return newTimeline; } } @Override public Map getSegmentLoadInfos() { - // map - } - - public Set getLoadedSegmentIds() - { - return segmentLoadInfos.keySet(); + return CollectionUtils.mapValues(getSelectors(), ServerSelector::toSegmentLoadInfo); } @Override @@ -421,68 +116,4 @@ public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) { return baseView.isSegmentLoadedByServer(serverKey, segment); } - - private void runTimelineCallbacks(final Function function) - { - for (Map.Entry entry : timelineCallbacks.entrySet()) { - entry.getValue().execute( - () -> { - if (CallbackAction.UNREGISTER == function.apply(entry.getKey())) { - timelineCallbacks.remove(entry.getKey()); - } - } - ); - } - } - - @Override - public void registerTimelineCallback(Executor exec, TimelineCallback callback) - { - timelineCallbacks.put(callback, exec); - } - - @Override - public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) - { - throw new UnsupportedOperationException(); - } - - @Override - public void registerSegmentCallback(Executor exec, SegmentCallback callback) - { - baseView.registerSegmentCallback(exec, callback); - } - - @Override - public Optional> getTimeline(DataSourceAnalysis analysis) - { - final TableDataSource table = - analysis.getBaseTableDataSource() - .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); - - synchronized (lock) { - return Optional.ofNullable(timelines2.get(table.getName())); - } - } - - @Override - public List getDruidServers() - { - return clients.values().stream() - .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) - .collect(Collectors.toList()); - } - - @Override - public QueryRunner getQueryRunner(DruidServer server) - { - synchronized (lock) { - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - log.error("No QueryRunner found for server name[%s].", server.getName()); - return null; - } - return queryableDruidServer.getQueryRunner(); - } - } } diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index 812f35d4622e..a56363afee83 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -48,7 +48,7 @@ public interface CoordinatorClient */ ListenableFuture> fetchUsedSegments(String dataSource, List intervals); - ListenableFuture> fetchDatasourceSchema(List datasources); + ListenableFuture> fetchDatasourceSchema(Set datasources); /** * Returns a new instance backed by a ServiceClient which follows the provided retryPolicy diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index dae00c73bea0..2a9e1891f086 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -26,17 +26,16 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler; -import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; import org.apache.druid.rpc.ServiceRetryPolicy; +import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentStatusInCluster; import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; -import java.util.Iterator; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -111,10 +110,13 @@ public ListenableFuture> fetchUsedSegments(String dataSource, } @Override - public ListenableFuture> fetchDatasourceSchema(List datasources) + public ListenableFuture> fetchDatasourceSchema(Set datasources) { final String path = "/druid/coordinator/v1/metadata/datasourceSchema"; - + if (null == datasources) + { + datasources = new HashSet<>(); + } return FutureUtils.transform( client.asyncRequest( new RequestBuilder(HttpMethod.POST, path) diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 8bc97a7b6efa..6879600878e2 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -19,10 +19,8 @@ package org.apache.druid.client.selector; -import com.google.common.collect.Sets; import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap; import org.apache.druid.client.DataSegmentInterner; -import org.apache.druid.client.ImmutableSegmentLoadInfo; import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunner; @@ -222,6 +220,12 @@ public boolean hasData() public SegmentLoadInfo toSegmentLoadInfo() { List allServers = getAllServers(); - return new SegmentLoadInfo(segment.get(), Sets.newConcurrentHashSet(allServers)); + SegmentLoadInfo segmentLoadInfo = new SegmentLoadInfo(segment.get()); + + for (DruidServerMetadata druidServerMetadata : allServers) { + segmentLoadInfo.addServer(druidServerMetadata); + } + + return segmentLoadInfo; } } diff --git a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java index 0b1468d0f129..0519445debfd 100644 --- a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java +++ b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java @@ -33,11 +33,6 @@ import java.util.List; import java.util.Set; -/** - * The difference between this class and org.apache.druid.sql.calcite.schema.MetadataSegmentView is that this class - * resides in Coordinator's memory, while org.apache.druid.sql.calcite.schema.MetadataSegmentView resides in Broker's - * memory. - */ public interface SegmentsMetadataManager { void startPollingDatabasePeriodically(); diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index c59ee23a77b2..c5d22c087794 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -7,10 +7,8 @@ public class SegmentMetadataCacheConfig { @JsonProperty private boolean awaitInitializationOnStart = true; - @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); - @JsonProperty private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 70ad14f625e6..ea92c4e8471a 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,12 +30,12 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; -import org.apache.druid.client.Alpha; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; +import org.apache.druid.client.CoordinatorInventoryView; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableSegmentLoadInfo; -import org.apache.druid.client.selector.ServerSelector; +import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.DateTimes; @@ -102,7 +102,7 @@ public class DataSourcesResource private static final Logger log = new Logger(DataSourcesResource.class); private static final long DEFAULT_LOADSTATUS_INTERVAL_OFFSET = 14 * 24 * 60 * 60 * 1000; - private final Alpha serverInventoryView; + private final CoordinatorInventoryView serverInventoryView; private final SegmentsMetadataManager segmentsMetadataManager; private final MetadataRuleManager metadataRuleManager; private final OverlordClient overlordClient; @@ -111,7 +111,7 @@ public class DataSourcesResource @Inject public DataSourcesResource( - Alpha serverInventoryView, + CoordinatorInventoryView serverInventoryView, SegmentsMetadataManager segmentsMetadataManager, MetadataRuleManager metadataRuleManager, @Nullable OverlordClient overlordClient, @@ -487,13 +487,13 @@ public Response getDatasourceLoadstatus( private SegmentsLoadStatistics computeSegmentLoadStatistics(Iterable segments) { - Set loadedSegmentIds = serverInventoryView.getLoadedSegmentIds(); + Map segmentLoadInfos = serverInventoryView.getSegmentLoadInfos(); int numPublishedSegments = 0; int numUnavailableSegments = 0; int numLoadedSegments = 0; for (DataSegment segment : segments) { numPublishedSegments++; - if (!loadedSegmentIds.contains(segment.getId())) { + if (!segmentLoadInfos.containsKey(segment.getId())) { numUnavailableSegments++; } else { numLoadedSegments++; @@ -820,7 +820,7 @@ public Response getServedSegmentsInInterval( @QueryParam("partial") final boolean partial ) { - TimelineLookup timeline = serverInventoryView.getTimeline( + TimelineLookup timeline = serverInventoryView.getTimeline( new TableDataSource(dataSourceName) ); final Interval theInterval = Intervals.of(interval.replace('_', '/')); @@ -833,19 +833,19 @@ public Response getServedSegmentsInInterval( } private Iterable prepareServedSegmentsInInterval( - TimelineLookup dataSourceServingTimeline, + TimelineLookup dataSourceServingTimeline, Interval interval ) { - Iterable> lookup = + Iterable> lookup = dataSourceServingTimeline.lookupWithIncompletePartitions(interval); return FunctionalIterable .create(lookup) .transformCat( - (TimelineObjectHolder input) -> + (TimelineObjectHolder input) -> Iterables.transform( input.getObject(), - (PartitionChunk chunk) -> chunk.getObject().toImmutableSegmentLoadInfo() + (PartitionChunk chunk) -> chunk.getObject().toImmutableSegmentLoadInfo() ) ); } @@ -885,7 +885,7 @@ public Response isHandOffComplete( return Response.ok(true).build(); } - TimelineLookup timeline = serverInventoryView.getTimeline( + TimelineLookup timeline = serverInventoryView.getTimeline( new TableDataSource(dataSourceName) ); if (timeline == null) { diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 75475eac6f9b..85c312f2b04e 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -37,12 +37,13 @@ import com.google.inject.util.Providers; import org.apache.curator.framework.CuratorFramework; import org.apache.druid.audit.AuditManager; -import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; -import org.apache.druid.client.Alpha; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.HttpServerInventoryViewResource; +import org.apache.druid.client.CoordinatorInventoryView; import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.TimelineAwareCoordinatorServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; @@ -192,12 +193,6 @@ protected List getModules() List modules = new ArrayList<>(); modules.add(JettyHttpClientModule.global()); - modules.add(new LegacyBrokerParallelMergeConfigModule()); - modules.add(new QueryRunnerFactoryModule()); - modules.add(new SegmentWranglerModule()); - modules.add(new QueryableModule()); - modules.add(new BrokerProcessingModule()); - modules.add(new JoinableFactoryModule()); modules.add( new Module() @@ -215,8 +210,6 @@ public void configure(Binder binder) binder.bind(MetadataStorage.class).toProvider(MetadataStorageProvider.class); - binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); - JsonConfigProvider.bind(binder, SegmentsMetadataManagerConfig.CONFIG_PREFIX, SegmentsMetadataManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.lookups", LookupCoordinatorManagerConfig.class); @@ -227,13 +220,6 @@ public void configure(Binder binder) "druid.coordinator.balancer.cachingCost", CachingCostBalancerStrategyConfig.class ); - JsonConfigProvider.bind(binder, "druid.coordinator.segment", SegmentMetadataCacheConfig.class); - JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.segment", BrokerSegmentWatcherConfig.class); binder.bind(RedirectFilter.class).in(LazySingleton.class); if (beOverlord) { @@ -242,6 +228,13 @@ public void configure(Binder binder) binder.bind(RedirectInfo.class).to(CoordinatorRedirectInfo.class).in(LazySingleton.class); } + if (isSegmentMetadataCacheEnabled()) { + binder.install(new SegmentMetadataCacheModule()); + } else { + binder.bind(CoordinatorInventoryView.class).to(CoordinatorServerView.class).in(LazySingleton.class); + LifecycleModule.register(binder, CoordinatorServerView.class); + } + binder.bind(SegmentsMetadataManager.class) .toProvider(SegmentsMetadataManagerProvider.class) .in(ManageLifecycle.class); @@ -255,17 +248,12 @@ public void configure(Binder binder) .in(ManageLifecycle.class); binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class); - binder.bind(CachingClusteredClient.class).in(LazySingleton.class); - binder.bind(Alpha.class); - binder.bind(TimelineServerView.class).to(Alpha.class).in(LazySingleton.class); + binder.bind(DruidCoordinator.class); - LifecycleModule.register(binder, Alpha.class); LifecycleModule.register(binder, MetadataStorage.class); LifecycleModule.register(binder, DruidCoordinator.class); - LifecycleModule.register(binder, SegmentMetadataCache.class); - binder.bind(JettyServerInitializer.class) .to(CoordinatorJettyServerInitializer.class); @@ -530,4 +518,33 @@ public Supplier> get() }; } } + + private static class SegmentMetadataCacheModule implements Module + { + @Override + public void configure(Binder binder) + { + binder.install(new LegacyBrokerParallelMergeConfigModule()); + binder.install(new QueryRunnerFactoryModule()); + binder.install(new SegmentWranglerModule()); + binder.install(new QueryableModule()); + binder.install(new BrokerProcessingModule()); + binder.install(new JoinableFactoryModule()); + + JsonConfigProvider.bind(binder, "druid.coordinator.segment", SegmentMetadataCacheConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); + + binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); + binder.bind(CachingClusteredClient.class).in(LazySingleton.class); + binder.bind(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); + binder.bind(CoordinatorInventoryView.class).to(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); + binder.bind(TimelineServerView.class).to(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); + LifecycleModule.register(binder, TimelineAwareCoordinatorServerView.class); + LifecycleModule.register(binder, SegmentMetadataCache.class); + } + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index d7a059710893..9219706940fe 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -6,13 +6,13 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig { @JsonProperty - private boolean metadataSegmentCacheEnable = false; + private boolean metadataSegmentCacheEnable = true; @JsonProperty private long metadataSegmentPollPeriod = 60000; @JsonProperty - private boolean useSegmentMetadataCache = false; + private boolean segmentMetadataCacheEnabled = false; public boolean isMetadataSegmentCacheEnable() { @@ -24,8 +24,8 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } - public boolean isUseSegmentMetadataCache() + public boolean isSegmentMetadataCacheEnabled() { - return useSegmentMetadataCache; + return segmentMetadataCacheEnabled; } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index 8b737ba9eb48..bb1d4f4ec6ff 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -25,8 +25,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableSortedSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.Uninterruptibles; import com.google.inject.Inject; import org.apache.druid.client.BrokerSegmentWatcherConfig; @@ -45,35 +43,32 @@ import org.apache.druid.java.util.common.lifecycle.LifecycleStart; import org.apache.druid.java.util.common.lifecycle.LifecycleStop; import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DatasourceSchema; -import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; - import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; - - - /** * This class polls the Coordinator in background to keep the latest published segments. * Provides {@link #getSegmentMetadata()} for others to get segments in metadata store. * - * The difference between this class and {@link SegmentsMetadataManager} is that this class resides - * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In - * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the + * This class polls the data from {@link SegmentsMetadataManager} object in the memory of the * currently leading Coordinator via HTTP queries. */ @ManageLifecycle @@ -92,11 +87,16 @@ public class BrokerSegmentMetadataView private final boolean useSegmentMetadataCache; + private final boolean includeRealtimeSegments; + + private final boolean pollDsSchema; + /** * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and * sys.segments queries return the segments in sorted order based on segmentId. * - * Volatile since this reference is reassigned in {@code poll()} and then read in {@code getPublishedSegments()} + * Volatile since this reference is reassigned in {@code pollSegmentMetadata()} + * and then read in {@code getSegmentMetadata()} * from other threads. */ @MonotonicNonNull @@ -129,21 +129,25 @@ public BrokerSegmentMetadataView( final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder, final BrokerServerView brokerServerView, final BrokerSegmentMetadataCache segmentMetadataCache - ) + ) { - Preconditions.checkNotNull(config, "SegmentMetadataCacheConfig"); + Preconditions.checkNotNull(config, "BrokerSegmentMetadataCacheConfig"); this.druidLeaderClient = druidLeaderClient; this.objectMapper = objectMapper; this.coordinatorClient = coordinatorClient; this.segmentWatcherConfig = segmentWatcherConfig; + this.isMetadataSegmentCacheEnabled = config.isMetadataSegmentCacheEnable(); + this.useSegmentMetadataCache = config.isSegmentMetadataCacheEnabled(); + this.includeRealtimeSegments = !useSegmentMetadataCache; + this.pollDsSchema = !useSegmentMetadataCache; + this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); - this.scheduledExec = Execs.scheduledSingleThreaded("MetadataSegmentView-Cache--%d"); + this.scheduledExec = Execs.scheduledSingleThreaded("SegmentMetadataView-Cache--%d"); this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() .expireAfterAccess(10, TimeUnit.MINUTES) .build(); this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; - this.useSegmentMetadataCache = config.isUseSegmentMetadataCache(); this.brokerServerView = brokerServerView; this.segmentMetadataCache = segmentMetadataCache; } @@ -155,7 +159,7 @@ public void start() throw new ISE("can't start."); } try { - if (isMetadataSegmentCacheEnabled || !useSegmentMetadataCache) { + if (isMetadataSegmentCacheEnabled || pollDsSchema) { scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); } lifecycleLock.started(); @@ -174,70 +178,158 @@ public void stop() throw new ISE("can't stop."); } log.info("MetadataSegmentView is stopping."); - if (isMetadataSegmentCacheEnabled) { + if (isMetadataSegmentCacheEnabled || pollDsSchema) { scheduledExec.shutdown(); } log.info("MetadataSegmentView Stopped."); } - private void pollSegmentMetadata() + protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) { - log.info("Polling segment metadata from coordinator"); + if (useSegmentMetadataCache) { + return segmentMetadataCache.getPhysicalDatasourceMetadata(name); + } + return datasourceSchemaMap.get(name); + } - segmentMetadata = fetchSegmentMetadata(); - segmentMetadataCachePopulated.countDown(); + protected Set getDatasourceNames() + { + if (useSegmentMetadataCache) { + return segmentMetadataCache.getDatasourceNames(); + } + return datasourceSchemaMap.keySet(); } - private ImmutableSortedSet fetchSegmentMetadata() + protected Iterator getSegmentTableView() { + if (useSegmentMetadataCache) { + return getSegmentTableViewFromCoordinatorAndSmc(); + } + return getSegmentTableViewFromCoordinator(); + } + + private Iterator getSegmentTableViewFromCoordinatorAndSmc() { - final Iterator metadataSegments = - querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); + final ImmutableSortedSet publishedSegments = getSegmentMetadata(); + final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); - while (metadataSegments.hasNext()) { - final SegmentStatusInCluster segment = metadataSegments.next(); - final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); - Integer replicationFactor = segment.getReplicationFactor(); - if (replicationFactor == null) { - replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); - } else { - segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); + final List segmentsTableView = new ArrayList<>(); + + Set seenSegments = new HashSet<>(); + for (SegmentStatusInCluster segmentStatusInCluster : publishedSegments) + { + DataSegment segment = segmentStatusInCluster.getDataSegment(); + SegmentId segmentId = segment.getId(); + AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataMap.get(segmentId); + + long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; + if (availableSegmentMetadata != null) { + numReplicas = availableSegmentMetadata.getNumReplicas(); + numRows = availableSegmentMetadata.getNumRows(); + isAvailable = 1L; + isRealtime = availableSegmentMetadata.isRealtime(); } - final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( - interned, - segment.isOvershadowed(), - replicationFactor, - segment.getNumRows(), - true, - segment.isRealtime() + + SegmentTableView segmentTableView = new SegmentTableView( + segment, + isAvailable, + isRealtime, + numReplicas, + numRows, + segmentStatusInCluster.getReplicationFactor(), + segmentStatusInCluster.isOvershadowed(), + true ); - builder.add(segmentStatusInCluster); + seenSegments.add(segmentId); + segmentsTableView.add((segmentTableView)); } - return builder.build(); + + for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) + { + if (seenSegments.contains(availableSegmentMetadataEntry.getKey())) { + continue; + } + AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataEntry.getValue(); + SegmentTableView segmentTableView = new SegmentTableView( + availableSegmentMetadata.getSegment(), + 1L, + availableSegmentMetadata.isRealtime(), + availableSegmentMetadata.getNumReplicas(), + availableSegmentMetadata.getNumRows(), + null, + false, + false + ); + segmentsTableView.add(segmentTableView); + } + + return segmentsTableView.iterator(); + } + + private Iterator getSegmentTableViewFromCoordinator() + { + final ImmutableSortedSet allSegments = getSegmentMetadata(); + final Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); + + final List segmentsTableView = new ArrayList<>(); + + for (SegmentStatusInCluster segmentStatusInCluster : allSegments) { + SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); + ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); + long numReplicas = 0L, isAvailable = 0L, numRows = 0L; + if (null != serverSelector) { + numReplicas = serverSelector.getAllServers().size(); + isAvailable = 1L; + } + if (null != segmentStatusInCluster.getNumRows()) + { + numRows = segmentStatusInCluster.getNumRows(); + } + + SegmentTableView segmentTableView = new SegmentTableView( + segmentStatusInCluster.getDataSegment(), + isAvailable, + segmentStatusInCluster.isRealtime(), + numReplicas, + numRows, + segmentStatusInCluster.getReplicationFactor(), + segmentStatusInCluster.isOvershadowed(), + segmentStatusInCluster.isPublished() + ); + segmentsTableView.add(segmentTableView); + } + + return segmentsTableView.iterator(); } private void pollDatasourceSchema() { log.info("Polling datasource schema from coordinator."); - Set datasources = useSegmentMetadataCache ? segmentMetadataCache.getDatasourceNames() : brokerServerView.getDatasourceNames(); + + Set watchedDatasources = segmentWatcherConfig.getWatchedDataSources(); Map physicalDatasourceMetadataMap = new HashMap<>(); - for (List partition : Iterables.partition(datasources, 100)) { - // retain watched datasources - List datasourceSchemas = FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema( - partition), true); + List datasourceSchemas = FutureUtils.getUnchecked( + coordinatorClient.fetchDatasourceSchema(watchedDatasources), true); - for (DatasourceSchema datasourceSchema : datasourceSchemas) { - physicalDatasourceMetadataMap.put( - datasourceSchema.getDatasource(), - physicalDatasourceMetadataBuilder.build(datasourceSchema.getDatasource(), datasourceSchema.getRowSignature())); - } + for (DatasourceSchema datasourceSchema : datasourceSchemas) { + physicalDatasourceMetadataMap.put( + datasourceSchema.getDatasource(), + physicalDatasourceMetadataBuilder.build(datasourceSchema.getDatasource(), datasourceSchema.getRowSignature()) + ); } this.datasourceSchemaMap = physicalDatasourceMetadataMap; } + private void pollSegmentMetadata() + { + log.info("Polling segment metadata from coordinator"); + + segmentMetadata = fetchSegmentMetadata(); + segmentMetadataCachePopulated.countDown(); + } + ImmutableSortedSet getSegmentMetadata() { if (isMetadataSegmentCacheEnabled) { @@ -248,65 +340,32 @@ ImmutableSortedSet getSegmentMetadata() } } - protected SystemSchema.SegmentsTable.SegmentTableView getSegmentTableView() { - ImmutableSortedSet allSegmentMetadata = getSegmentMetadata(); - - log.info("logging polled segments from coordinator %s", allSegmentMetadata); - final ImmutableSortedSet.Builder publishedSegmentBuilder = ImmutableSortedSet.naturalOrder(); - - Map availableSegmentMetadataMap; - - if (useSegmentMetadataCache) { - availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - - for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { - if (segmentStatusInCluster.isPublished()) { - publishedSegmentBuilder.add(segmentStatusInCluster); - } - } - } else { - // build available segment metadata map by combining stuff from brokerServerView and numRows from data returned from coordinator - availableSegmentMetadataMap = new HashMap<>(); - Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); - // only look at watched ds, confirm if brokerServerView is also looking for watched ds - for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { - if (segmentStatusInCluster.isPublished()) { - publishedSegmentBuilder.add(segmentStatusInCluster); - } - SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); - if (!brokerSegmentMetadata.containsKey(segmentId)) { - // log and count ignored segments - continue; - } - ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); - - AvailableSegmentMetadata availableSegmentMetadata = - AvailableSegmentMetadata.builder( - segmentStatusInCluster.getDataSegment(), - segmentStatusInCluster.isRealtime(), - Sets.newHashSet(serverSelector.getAllServers()), - null, - segmentStatusInCluster.getNumRows() == null ? -1 : segmentStatusInCluster.getNumRows() - ) - .build(); - - availableSegmentMetadataMap.put(segmentId, availableSegmentMetadata); + private ImmutableSortedSet fetchSegmentMetadata() + { + final Iterator metadataSegments = + querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); + final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); + while (metadataSegments.hasNext()) { + final SegmentStatusInCluster segment = metadataSegments.next(); + final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); + Integer replicationFactor = segment.getReplicationFactor(); + if (replicationFactor == null) { + replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); + } else { + segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); } + final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( + interned, + segment.isOvershadowed(), + replicationFactor, + segment.getNumRows(), + true, + segment.isRealtime() + ); + builder.add(segmentStatusInCluster); } - - log.info("Logging Segment table view. availableSmMap [%s], published segments [%s]", availableSegmentMetadataMap, publishedSegmentBuilder); - return new SystemSchema.SegmentsTable.SegmentTableView(availableSegmentMetadataMap, publishedSegmentBuilder.build()); - } - - protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) - { - return useSegmentMetadataCache ? segmentMetadataCache.getPhysicalDatasourceMetadata(name) : datasourceSchemaMap.get(name); - } - - protected Set getDatasourceNames() - { - return useSegmentMetadataCache ? segmentMetadataCache.getDatasourceNames() : datasourceSchemaMap.keySet(); + return builder.build(); } // Note that coordinator must be up to get segments @@ -314,20 +373,21 @@ private JsonParserIterator querySegmentMetadata( Set watchedDataSources ) { - String query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"; + final StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"); + if (includeRealtimeSegments) { + queryBuilder.append("&includeRealtimeSegments"); + } if (watchedDataSources != null && !watchedDataSources.isEmpty()) { log.debug( "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); - final StringBuilder sb = new StringBuilder(); for (String ds : watchedDataSources) { - sb.append("datasources=").append(ds).append("&"); + queryBuilder.append("&datasources=").append(ds).append("&"); } - sb.setLength(sb.length() - 1); - query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&" + sb; + queryBuilder.setLength(queryBuilder.length() - 1); } return SystemSchema.getThingsFromLeaderNode( - query, + queryBuilder.toString(), new TypeReference() { }, @@ -347,7 +407,7 @@ public void run() if (isMetadataSegmentCacheEnabled) { pollSegmentMetadata(); } - if (!useSegmentMetadataCache) { + if (pollDsSchema) { pollDatasourceSchema(); } final long pollEndTime = System.nanoTime(); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index 8b7d885ed64b..3003b99fbe1c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -19,9 +19,7 @@ package org.apache.druid.sql.calcite.schema; -import com.google.common.base.Preconditions; import com.google.inject.Binder; -import com.google.inject.Inject; import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; @@ -40,7 +38,7 @@ public class DruidCalciteSchemaModule implements Module { private static final String DRUID_SCHEMA_NAME = "druid"; private static final String INFORMATION_SCHEMA_NAME = "INFORMATION_SCHEMA"; - private static final String SEGMENT_METADATA_CACHE_DISABLED = "druid.sql.planner.disableSegmentMetadataCache"; + private static final String SEGMENT_METADATA_CACHE_ENABLED = "druid.sql.planner.segmentMetadataCacheEnabled"; static final String INCOMPLETE_SCHEMA = "INCOMPLETE_SCHEMA"; private final Properties properties; @@ -62,7 +60,7 @@ public void configure(Binder binder) .in(Scopes.SINGLETON); // BrokerSegmentMetadataCache needs to listen to changes for incoming segments - if (!isSegmentMetadataCacheDisabled()) { + if (isSegmentMetadataCacheEnabled()) { LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); } binder.bind(DruidSchema.class).in(Scopes.SINGLETON); @@ -85,8 +83,8 @@ private DruidSchemaCatalog getRootSchema(@Named(INCOMPLETE_SCHEMA) DruidSchemaCa return rootSchema; } - private boolean isSegmentMetadataCacheDisabled() + private boolean isSegmentMetadataCacheEnabled() { - return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_DISABLED)); + return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED, "true")); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 73610534493a..38491de49507 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -72,6 +72,7 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.DruidNode; +import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthenticationResult; @@ -110,10 +111,10 @@ public class SystemSchema extends AbstractSchema private static final String TASKS_TABLE = "tasks"; private static final String SUPERVISOR_TABLE = "supervisors"; - private static final Function> - SEGMENT_STATUS_IN_CLUSTER_RA_GENERATOR = segment -> + private static final Function> + SEGMENT_STATUS_IN_CLUSTER_RA_GENERATOR = segmentTableView -> Collections.singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply( - segment.getDataSegment().getDataSource()) + segmentTableView.getSegment().getDataSource()) ); private static final Function> SEGMENT_RA_GENERATOR = @@ -278,41 +279,12 @@ public TableType getJdbcTableType() @Override public Enumerable scan(DataContext root) { - final SegmentTableView segmentTableView = metadataView.getSegmentTableView(); - - final Map availableSegmentMetadata = - segmentTableView.getAvailableSegmentMetadata(); - final Iterator> availableSegmentEntries = - availableSegmentMetadata.entrySet().iterator(); - - // in memory map to store segment data from available segments - final Map partialSegmentDataMap = - Maps.newHashMapWithExpectedSize(segmentTableView.totalSegmentCount()); - for (AvailableSegmentMetadata h : availableSegmentMetadata.values()) { - PartialSegmentData partialSegmentData = - new PartialSegmentData(IS_AVAILABLE_TRUE, h.isRealtime(), h.getNumReplicas(), h.getNumRows()); - partialSegmentDataMap.put(h.getSegment().getId(), partialSegmentData); - } - - // Get published segments from metadata segment cache (if enabled in SQL planner config), else directly from - // Coordinator. - final Iterator metadataStoreSegments = segmentTableView.getPublishedSegments(); - - final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(segmentTableView.totalSegmentCount()); + final Iterator segmentTableView = metadataView.getSegmentTableView(); - final FluentIterable publishedSegments = FluentIterable - .from(() -> getAuthorizedPublishedSegments(metadataStoreSegments, root)) + final FluentIterable allSegments = FluentIterable + .from(() -> getAuthorizedPublishedSegments(segmentTableView, root)) .transform(val -> { - final DataSegment segment = val.getDataSegment(); - segmentsAlreadySeen.add(segment.getId()); - final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(segment.getId()); - long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; - if (partialSegmentData != null) { - numReplicas = partialSegmentData.getNumReplicas(); - numRows = partialSegmentData.getNumRows(); - isAvailable = partialSegmentData.isAvailable(); - isRealtime = partialSegmentData.isRealtime(); - } + final DataSegment segment = val.getSegment(); try { return new Object[]{ segment.getId(), @@ -322,19 +294,21 @@ public Enumerable scan(DataContext root) segment.getSize(), segment.getVersion(), (long) segment.getShardSpec().getPartitionNum(), - numReplicas, - numRows, + val.getNumReplicas(), + val.getNumRows(), //is_active is true for published segments that are not overshadowed val.isOvershadowed() ? IS_ACTIVE_FALSE : IS_ACTIVE_TRUE, //is_published is true for published segments - IS_PUBLISHED_TRUE, - isAvailable, - isRealtime, + val.isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, + val.getIsAvailable(), + val.getIsRealtime(), val.isOvershadowed() ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE, segment.getShardSpec() == null ? null : jsonMapper.writeValueAsString(segment.getShardSpec()), segment.getDimensions() == null ? null : jsonMapper.writeValueAsString(segment.getDimensions()), segment.getMetrics() == null ? null : jsonMapper.writeValueAsString(segment.getMetrics()), - segment.getLastCompactionState() == null ? null : jsonMapper.writeValueAsString(segment.getLastCompactionState()), + segment.getLastCompactionState() == null + ? null + : jsonMapper.writeValueAsString(segment.getLastCompactionState()), // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. // This should be automatically updated in the next refesh with Coordinator. val.getReplicationFactor() == null ? REPLICATION_FACTOR_UNKNOWN : (long) val.getReplicationFactor() @@ -345,59 +319,12 @@ public Enumerable scan(DataContext root) } }); - final FluentIterable availableSegments = FluentIterable - .from(() -> getAuthorizedAvailableSegments( - availableSegmentEntries, - root - )) - .transform(val -> { - if (segmentsAlreadySeen.contains(val.getKey())) { - return null; - } - final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(val.getKey()); - final long numReplicas = partialSegmentData == null ? 0L : partialSegmentData.getNumReplicas(); - try { - return new Object[]{ - val.getKey(), - val.getKey().getDataSource(), - val.getKey().getInterval().getStart().toString(), - val.getKey().getInterval().getEnd().toString(), - val.getValue().getSegment().getSize(), - val.getKey().getVersion(), - (long) val.getValue().getSegment().getShardSpec().getPartitionNum(), - numReplicas, - val.getValue().getNumRows(), - // is_active is true for unpublished segments iff they are realtime - val.getValue().isRealtime() /* is_active */, - // is_published is false for unpublished segments - IS_PUBLISHED_FALSE, - // is_available is assumed to be always true for segments announced by historicals or realtime tasks - IS_AVAILABLE_TRUE, - val.getValue().isRealtime(), - IS_OVERSHADOWED_FALSE, - // there is an assumption here that unpublished segments are never overshadowed - val.getValue().getSegment().getShardSpec() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getShardSpec()), - val.getValue().getSegment().getDimensions() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getDimensions()), - val.getValue().getSegment().getMetrics() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getMetrics()), - null, // unpublished segments from realtime tasks will not be compacted yet - REPLICATION_FACTOR_UNKNOWN // If the segment is unpublished, we won't have this information yet. - }; - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); - - final Iterable allSegments = Iterables.unmodifiableIterable( - Iterables.concat(publishedSegments, availableSegments) - ); return Linq4j.asEnumerable(allSegments).where(Objects::nonNull); - } - private Iterator getAuthorizedPublishedSegments( - Iterator it, + private Iterator getAuthorizedPublishedSegments( + Iterator it, DataContext root ) { @@ -406,7 +333,7 @@ private Iterator getAuthorizedPublishedSegments( "authenticationResult in dataContext" ); - final Iterable authorizedSegments = AuthorizationUtils + final Iterable authorizedSegments = AuthorizationUtils .filterAuthorizedResources( authenticationResult, () -> it, @@ -443,6 +370,79 @@ private Iterator> getAuthorizedAvaila } protected static class SegmentTableView + { + private final DataSegment segment; + private final long isAvailable; + private final long isRealtime; + private final long numReplicas; + private final long numRows; + private final Integer replicationFactor; + private final boolean isOvershadowed; + private final boolean isPublished; + + public SegmentTableView( + DataSegment segment, + long isAvailable, + long isRealtime, + long numReplicas, + long numRows, + Integer replicationFactor, + boolean isOvershadowed, + boolean isPublished + ) + { + this.segment = segment; + this.isAvailable = isAvailable; + this.isRealtime = isRealtime; + this.numReplicas = numReplicas; + this.numRows = numRows; + this.replicationFactor = replicationFactor; + this.isOvershadowed = isOvershadowed; + this.isPublished = isPublished; + } + + public DataSegment getSegment() + { + return segment; + } + + public long getIsAvailable() + { + return isAvailable; + } + + public long getIsRealtime() + { + return isRealtime; + } + + public long getNumReplicas() + { + return numReplicas; + } + + public long getNumRows() + { + return numRows; + } + + public Integer getReplicationFactor() + { + return replicationFactor; + } + + public boolean isOvershadowed() + { + return isOvershadowed; + } + + public boolean isPublished() + { + return isPublished; + } + } + + protected static class SegmentTableView1 { // pass the stitched private final Map availableSegmentMetadata; @@ -450,7 +450,7 @@ protected static class SegmentTableView private final int totalSegmentsCount; - public SegmentTableView( + public SegmentTableView1( Map availableSegmentMetadata, ImmutableSortedSet publishedSegments ) @@ -489,7 +489,6 @@ public PartialSegmentData( final long numReplicas, final long numRows ) - { this.isAvailable = isAvailable; this.isRealtime = isRealtime; From 105bda960390fdbae9592f00893ecd145dd630f3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 6 Sep 2023 14:43:35 +0530 Subject: [PATCH 06/82] Move schema querying logic to BrokerSegmentMetadataCache --- .../timeline/SegmentStatusInCluster.java | 30 ++-- .../metadata/SegmentMetadataCache.java | 16 +- .../druid/server/http/MetadataResource.java | 88 +++++---- .../org/apache/druid/cli/CliCoordinator.java | 1 + .../schema/BrokerSegmentMetadataCache.java | 84 ++++++++- .../BrokerSegmentMetadataCacheConfig.java | 7 - .../schema/BrokerSegmentMetadataView.java | 170 +++++------------- .../druid/sql/calcite/schema/DruidSchema.java | 13 +- .../sql/calcite/schema/SystemSchema.java | 27 +-- .../schema/SegmentMetadataCacheTest.java | 4 +- 10 files changed, 216 insertions(+), 224 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index 263f1619c6f3..bccfa764726b 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -58,21 +58,21 @@ public class SegmentStatusInCluster implements Comparable dataSources, @QueryParam("includeOvershadowedStatus") final @Nullable String includeOvershadowedStatus, - @QueryParam("includeOvershadowedStatus") final @Nullable String includeRealtime + @QueryParam("includeRealtimeSegments") final @Nullable String includeRealtimeSegments ) { if (includeOvershadowedStatus != null) { - return getAllUsedSegmentsWithAdditionalDetails(req, dataSources); + return getAllUsedSegmentsWithAdditionalDetails(req, dataSources, includeRealtimeSegments); } Collection dataSourcesWithUsedSegments = @@ -180,7 +181,8 @@ public Response getAllUsedSegments( private Response getAllUsedSegmentsWithAdditionalDetails( HttpServletRequest req, - @Nullable Set dataSources + @Nullable Set dataSources, + String includeRealtimeSegments ) { DataSourcesSnapshot dataSourcesSnapshot = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments(); @@ -200,44 +202,61 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .map(segment -> { // The replication factor for unloaded segments is 0 as they will be unloaded soon boolean isOvershadowed = overshadowedSegments.contains(segment); - AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( - segment.getDataSource(), - segment.getId() - ); + Long numRows = null; + if (null != segmentMetadataCache) { + AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( + segment.getDataSource(), + segment.getId() + ); + if (null != availableSegmentMetadata) { + numRows = availableSegmentMetadata.getNumRows(); + } + } Integer replicationFactor = isOvershadowed ? (Integer) 0 : coordinator.getReplicationFactor(segment.getId()); - Long numRows = (null != availableSegmentMetadata) ? availableSegmentMetadata.getNumRows() : null; segmentAlreadySeen.add(segment.getId()); - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, 20L, true, 0L); - }).peek(v -> log.info("peeking into first stream segmentseen [%s], id [%s] isPublished [%s]", segmentAlreadySeen, v.getDataSegment().getId(), v.isPublished())); + return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, 20L, true); + }).peek(v -> log.info("peeking into first stream segmentseen [%s], id [%s]", segmentAlreadySeen, v.getDataSegment().getId())); - log.info("printing the content in smc cache %s", segmentMetadataCache.getSegmentMetadataSnapshot().values()); + Stream finalSegments = segmentStatus; + + if (null != includeRealtimeSegments && segmentMetadataCache != null) { + log.info("printing the content in smc cache %s", segmentMetadataCache.getSegmentMetadataSnapshot().values()); + final Stream realtimeSegmentStatus = segmentMetadataCache + .getSegmentMetadataSnapshot() + .values() + .stream() + .peek(v -> log.info( + "peeking into second stream (first) segmentseen [%s], id [%s]", + segmentAlreadySeen, + v.getSegment().getId() + )) + .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment() + .getId())) + .map(availableSegmentMetadata -> { + return new SegmentStatusInCluster( + availableSegmentMetadata.getSegment(), + false, + (int) availableSegmentMetadata.getNumReplicas(), + /**availableSegmentMetadata.getNumRows(), **/30L, + availableSegmentMetadata.isRealtime() != 0 + ); + }) + .peek(v -> log.info("peeking into second stream (second) segmentseen [%s], id [%s]", + segmentAlreadySeen, + v.getDataSegment().getId() + )); + + finalSegments = Stream.concat(segmentStatus, realtimeSegmentStatus) + .peek(v -> log.info("combined stream element %s", v)); + } - final Stream realtimeSegmentStatus = segmentMetadataCache - .getSegmentMetadataSnapshot() - .values() - .stream() - .peek(v -> log.info("peeking into second stream (first) segmentseen [%s], id [%s]", segmentAlreadySeen, v.getSegment().getId())) - .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment().getId())) - .map(availableSegmentMetadata -> { - return new SegmentStatusInCluster( - availableSegmentMetadata.getSegment(), - false, - (int) availableSegmentMetadata.getNumReplicas(), - /**availableSegmentMetadata.getNumRows(), **/ 30L, - false, - availableSegmentMetadata.isRealtime() - ); - }).peek(v -> log.info("peeking into second stream (second) segmentseen [%s], id [%s]", segmentAlreadySeen, v.getDataSegment().getId()));; - - - Stream combined = Stream.concat(segmentStatus, realtimeSegmentStatus).peek(v -> log.info("combined stream element %s", v)); final Function> raGenerator = segment -> Collections .singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSegment().getDataSource())); final Iterable authorizedSegments = AuthorizationUtils.filterAuthorizedResources( req, - combined::iterator, + finalSegments::iterator, raGenerator, authorizerMapper ); @@ -347,6 +366,9 @@ public Response getDatasourceSchema( List datasources ) { + if (null == segmentMetadataCache) { + return Response.status(Response.Status.NOT_FOUND).build(); + } Map datasourceSchemaMap = segmentMetadataCache.getDatasourceSchemaMap(); datasourceSchemaMap.keySet().retainAll(datasources); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 85c312f2b04e..6c16684a2e93 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -233,6 +233,7 @@ public void configure(Binder binder) } else { binder.bind(CoordinatorInventoryView.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); + binder.bind(SegmentMetadataCache.class).toProvider(Providers.of(null)); } binder.bind(SegmentsMetadataManager.class) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 09f252cc3e20..a7f47d6c5fde 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -1,9 +1,11 @@ package org.apache.druid.sql.calcite.schema; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import com.google.inject.Inject; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; @@ -13,7 +15,11 @@ import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.timeline.SegmentId; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -25,20 +31,23 @@ @ManageLifecycle public class BrokerSegmentMetadataCache extends SegmentMetadataCache { - private static final EmittingLogger log = new EmittingLogger(SegmentMetadataCache.class); + private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataCache.class); private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final CoordinatorClient coordinatorClient; + @Inject public BrokerSegmentMetadataCache( - QueryLifecycleFactory queryLifecycleFactory, - TimelineServerView serverView, - SegmentMetadataCacheConfig config, - Escalator escalator, - InternalQueryConfig internalQueryConfig, - ServiceEmitter emitter, - PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder + final QueryLifecycleFactory queryLifecycleFactory, + final TimelineServerView serverView, + final SegmentMetadataCacheConfig config, + final Escalator escalator, + final InternalQueryConfig internalQueryConfig, + final ServiceEmitter emitter, + final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder, + final CoordinatorClient coordinatorClient ) { super( @@ -50,6 +59,63 @@ public BrokerSegmentMetadataCache( emitter ); this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; + this.coordinatorClient = coordinatorClient; + } + + @Override + public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException + { + Set dataSourcesToQuery = new HashSet<>(); + + segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); + + Map polledDataSourceSchema = new HashMap<>(); + + // Fetch dataSource schema from the Coordinator + try { + FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema(dataSourcesToQuery), true) + .forEach(item -> polledDataSourceSchema.put( + item.getDatasource(), + physicalDatasourceMetadataBuilder.build( + item.getDatasource(), + item.getRowSignature() + ) + )); + } catch (Exception e) { + log.error("Exception querying coordinator for schema"); + } + + log.info("Queried ds schema are [%s]", polledDataSourceSchema); + + tables.putAll(polledDataSourceSchema); + + // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. + segmentsToRefresh.forEach(segment -> { + if (polledDataSourceSchema.containsKey(segment.getDataSource())) { + segmentsToRefresh.remove(segment); + } + }); + + // Refresh the segments. + final Set refreshed = refreshSegments(segmentsToRefresh); + + synchronized (getLock()) { + // Add missing segments back to the refresh list. + getSegmentsNeedingRefresh().addAll(Sets.difference(segmentsToRefresh, refreshed)); + + // Compute the list of dataSources to rebuild tables for. + dataSourcesToRebuild.addAll(getDataSourcesNeedingRebuild()); + refreshed.forEach(segment -> dataSourcesToRebuild.add(segment.getDataSource())); + + // Remove those dataSource for which we received schema from the Coordinator. + dataSourcesToRebuild.removeAll(polledDataSourceSchema.keySet()); + getDataSourcesNeedingRebuild().clear(); + } + + // Rebuild the dataSources. + for (String dataSource : dataSourcesToRebuild) { + rebuildDatasource(dataSource); + } } @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 9219706940fe..7f4041fb0b03 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -11,9 +11,6 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig @JsonProperty private long metadataSegmentPollPeriod = 60000; - @JsonProperty - private boolean segmentMetadataCacheEnabled = false; - public boolean isMetadataSegmentCacheEnable() { return metadataSegmentCacheEnable; @@ -24,8 +21,4 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } - public boolean isSegmentMetadataCacheEnabled() - { - return segmentMetadataCacheEnabled; - } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index bb1d4f4ec6ff..9eeafcd4f8c4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -32,9 +32,7 @@ import org.apache.druid.client.DataSegmentInterner; import org.apache.druid.client.JsonParserIterator; import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.client.coordinator.CoordinatorClient; import org.apache.druid.client.selector.ServerSelector; -import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.concurrent.LifecycleLock; import org.apache.druid.discovery.DruidLeaderClient; import org.apache.druid.guice.ManageLifecycle; @@ -45,16 +43,13 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; -import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -76,20 +71,13 @@ public class BrokerSegmentMetadataView { private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataView.class); - private final CoordinatorClient coordinatorClient; private final BrokerSegmentWatcherConfig segmentWatcherConfig; private final DruidLeaderClient druidLeaderClient; private final ObjectMapper objectMapper; - private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; private final boolean isMetadataSegmentCacheEnabled; - private final boolean useSegmentMetadataCache; - - private final boolean includeRealtimeSegments; - - private final boolean pollDsSchema; /** * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and @@ -102,8 +90,6 @@ public class BrokerSegmentMetadataView @MonotonicNonNull private volatile ImmutableSortedSet segmentMetadata = null; - private volatile Map datasourceSchemaMap = null; - private final BrokerServerView brokerServerView; private final BrokerSegmentMetadataCache segmentMetadataCache; @@ -123,10 +109,8 @@ public class BrokerSegmentMetadataView public BrokerSegmentMetadataView( final @Coordinator DruidLeaderClient druidLeaderClient, final ObjectMapper objectMapper, - final CoordinatorClient coordinatorClient, final BrokerSegmentWatcherConfig segmentWatcherConfig, final BrokerSegmentMetadataCacheConfig config, - final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder, final BrokerServerView brokerServerView, final BrokerSegmentMetadataCache segmentMetadataCache ) @@ -134,20 +118,15 @@ public BrokerSegmentMetadataView( Preconditions.checkNotNull(config, "BrokerSegmentMetadataCacheConfig"); this.druidLeaderClient = druidLeaderClient; this.objectMapper = objectMapper; - this.coordinatorClient = coordinatorClient; this.segmentWatcherConfig = segmentWatcherConfig; this.isMetadataSegmentCacheEnabled = config.isMetadataSegmentCacheEnable(); - this.useSegmentMetadataCache = config.isSegmentMetadataCacheEnabled(); - this.includeRealtimeSegments = !useSegmentMetadataCache; - this.pollDsSchema = !useSegmentMetadataCache; this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); this.scheduledExec = Execs.scheduledSingleThreaded("SegmentMetadataView-Cache--%d"); this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() .expireAfterAccess(10, TimeUnit.MINUTES) .build(); - this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; this.brokerServerView = brokerServerView; this.segmentMetadataCache = segmentMetadataCache; } @@ -159,12 +138,12 @@ public void start() throw new ISE("can't start."); } try { - if (isMetadataSegmentCacheEnabled || pollDsSchema) { + if (isMetadataSegmentCacheEnabled) { scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); } lifecycleLock.started(); - log.info("MetadataSegmentView Started. Configs isMetadataSegmentCacheEnabled [%s], useSegmentMetadataCache [%s]", - isMetadataSegmentCacheEnabled, useSegmentMetadataCache); + log.info("MetadataSegmentView Started. Configs isMetadataSegmentCacheEnabled [%s]", + isMetadataSegmentCacheEnabled); } finally { lifecycleLock.exitStart(); @@ -178,57 +157,47 @@ public void stop() throw new ISE("can't stop."); } log.info("MetadataSegmentView is stopping."); - if (isMetadataSegmentCacheEnabled || pollDsSchema) { + if (isMetadataSegmentCacheEnabled) { scheduledExec.shutdown(); } log.info("MetadataSegmentView Stopped."); } - protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) - { - if (useSegmentMetadataCache) { - return segmentMetadataCache.getPhysicalDatasourceMetadata(name); - } - return datasourceSchemaMap.get(name); - } - - protected Set getDatasourceNames() + protected Iterator getSegmentTableView() { - if (useSegmentMetadataCache) { - return segmentMetadataCache.getDatasourceNames(); - } - return datasourceSchemaMap.keySet(); - } - - protected Iterator getSegmentTableView() { - if (useSegmentMetadataCache) { - return getSegmentTableViewFromCoordinatorAndSmc(); - } - return getSegmentTableViewFromCoordinator(); - } - - private Iterator getSegmentTableViewFromCoordinatorAndSmc() - { - final ImmutableSortedSet publishedSegments = getSegmentMetadata(); + final ImmutableSortedSet segments = getSegmentMetadata(); final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - - final List segmentsTableView = new ArrayList<>(); + final Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); + final List segmentsTableViews = new ArrayList<>(); Set seenSegments = new HashSet<>(); - for (SegmentStatusInCluster segmentStatusInCluster : publishedSegments) + + for (SegmentStatusInCluster segmentStatusInCluster : segments) { DataSegment segment = segmentStatusInCluster.getDataSegment(); SegmentId segmentId = segment.getId(); AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataMap.get(segmentId); - long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; + long numReplicas = 0L, numRows = 0L, isAvailable = 0L; if (availableSegmentMetadata != null) { numReplicas = availableSegmentMetadata.getNumReplicas(); numRows = availableSegmentMetadata.getNumRows(); isAvailable = 1L; - isRealtime = availableSegmentMetadata.isRealtime(); + } else if (brokerSegmentMetadata.containsKey(segmentId)) { + ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); + numReplicas = serverSelector.getAllServers().size(); + isAvailable = 1L; + } + + // Prefer numRows & realtime status info returned from Coordinator. + if (null != segmentStatusInCluster.getNumRows()) + { + numRows = segmentStatusInCluster.getNumRows(); } + long isRealtime = Boolean.TRUE.equals(segmentStatusInCluster.isRealtime()) ? 1 : 0; + + SegmentTableView segmentTableView = new SegmentTableView( segment, isAvailable, @@ -236,11 +205,10 @@ private Iterator getSegmentTableViewFromCoordinatorAndSmc() numReplicas, numRows, segmentStatusInCluster.getReplicationFactor(), - segmentStatusInCluster.isOvershadowed(), - true + segmentStatusInCluster.isOvershadowed() ); seenSegments.add(segmentId); - segmentsTableView.add((segmentTableView)); + segmentsTableViews.add((segmentTableView)); } for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) @@ -256,70 +224,16 @@ private Iterator getSegmentTableViewFromCoordinatorAndSmc() availableSegmentMetadata.getNumReplicas(), availableSegmentMetadata.getNumRows(), null, - false, false ); - segmentsTableView.add(segmentTableView); - } - - return segmentsTableView.iterator(); - } - - private Iterator getSegmentTableViewFromCoordinator() - { - final ImmutableSortedSet allSegments = getSegmentMetadata(); - final Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); - - final List segmentsTableView = new ArrayList<>(); - - for (SegmentStatusInCluster segmentStatusInCluster : allSegments) { - SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); - ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); - long numReplicas = 0L, isAvailable = 0L, numRows = 0L; - if (null != serverSelector) { - numReplicas = serverSelector.getAllServers().size(); - isAvailable = 1L; - } - if (null != segmentStatusInCluster.getNumRows()) - { - numRows = segmentStatusInCluster.getNumRows(); - } - - SegmentTableView segmentTableView = new SegmentTableView( - segmentStatusInCluster.getDataSegment(), - isAvailable, - segmentStatusInCluster.isRealtime(), - numReplicas, - numRows, - segmentStatusInCluster.getReplicationFactor(), - segmentStatusInCluster.isOvershadowed(), - segmentStatusInCluster.isPublished() - ); - segmentsTableView.add(segmentTableView); + segmentsTableViews.add(segmentTableView); } - return segmentsTableView.iterator(); - } - - private void pollDatasourceSchema() - { - log.info("Polling datasource schema from coordinator."); - - Set watchedDatasources = segmentWatcherConfig.getWatchedDataSources(); - - Map physicalDatasourceMetadataMap = new HashMap<>(); - - List datasourceSchemas = FutureUtils.getUnchecked( - coordinatorClient.fetchDatasourceSchema(watchedDatasources), true); - - for (DatasourceSchema datasourceSchema : datasourceSchemas) { - physicalDatasourceMetadataMap.put( - datasourceSchema.getDatasource(), - physicalDatasourceMetadataBuilder.build(datasourceSchema.getDatasource(), datasourceSchema.getRowSignature()) - ); + log.info("Built the segment table view"); + for (SegmentTableView segmentTableView : segmentsTableViews) { + log.info("SegmentTableView is [%s]", segmentTableView); } - - this.datasourceSchemaMap = physicalDatasourceMetadataMap; + return segmentsTableViews.iterator(); } private void pollSegmentMetadata() @@ -346,8 +260,10 @@ private ImmutableSortedSet fetchSegmentMetadata() querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); + log.info("Polled segments from coordinator"); while (metadataSegments.hasNext()) { final SegmentStatusInCluster segment = metadataSegments.next(); + log.info("This is the polled segmentStatusInCluster %s", segment); final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); Integer replicationFactor = segment.getReplicationFactor(); if (replicationFactor == null) { @@ -360,23 +276,22 @@ private ImmutableSortedSet fetchSegmentMetadata() segment.isOvershadowed(), replicationFactor, segment.getNumRows(), - true, segment.isRealtime() ); + log.info("SegmentStatusInCluster %s", segmentStatusInCluster); builder.add(segmentStatusInCluster); } + return builder.build(); } - // Note that coordinator must be up to get segments private JsonParserIterator querySegmentMetadata( Set watchedDataSources ) { final StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"); - if (includeRealtimeSegments) { - queryBuilder.append("&includeRealtimeSegments"); - } + + //queryBuilder.append("&includeRealtimeSegments"); if (watchedDataSources != null && !watchedDataSources.isEmpty()) { log.debug( "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); @@ -386,8 +301,10 @@ private JsonParserIterator querySegmentMetadata( queryBuilder.setLength(queryBuilder.length() - 1); } + String query = queryBuilder.toString(); + log.info("query is %s", query); return SystemSchema.getThingsFromLeaderNode( - queryBuilder.toString(), + query, new TypeReference() { }, @@ -404,12 +321,7 @@ public void run() long delayMS = pollPeriodInMS; try { final long pollStartTime = System.nanoTime(); - if (isMetadataSegmentCacheEnabled) { - pollSegmentMetadata(); - } - if (pollDsSchema) { - pollDatasourceSchema(); - } + pollSegmentMetadata(); final long pollEndTime = System.nanoTime(); final long pollTimeNS = pollEndTime - pollStartTime; final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index 28a6ab252516..7f4f626e6ec6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite.schema; import org.apache.calcite.schema.Table; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.sql.calcite.table.DatasourceTable; import javax.inject.Inject; @@ -28,16 +29,16 @@ public class DruidSchema extends AbstractTableSchema { - private final BrokerSegmentMetadataView brokerSegmentMetadataView; + private final BrokerSegmentMetadataCache segmentMetadataCache; private final DruidSchemaManager druidSchemaManager; @Inject public DruidSchema( - final BrokerSegmentMetadataView brokerSegmentMetadataView, + final BrokerSegmentMetadataCache segmentMetadataCache, final DruidSchemaManager druidSchemaManager ) { - this.brokerSegmentMetadataView = brokerSegmentMetadataView; + this.segmentMetadataCache = segmentMetadataCache; if (druidSchemaManager != null && !(druidSchemaManager instanceof NoopDruidSchemaManager)) { this.druidSchemaManager = druidSchemaManager; } else { @@ -51,9 +52,9 @@ public Table getTable(String name) if (druidSchemaManager != null) { return druidSchemaManager.getTable(name); } else { - DatasourceTable.PhysicalDatasourceMetadata dsMetadata = brokerSegmentMetadataView.getDatasource(name); + DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentMetadataCache.getPhysicalDatasourceMetadata(name); return dsMetadata == null ? null : new DatasourceTable(dsMetadata); -} + } } @Override @@ -62,7 +63,7 @@ public Set getTableNames() if (druidSchemaManager != null) { return druidSchemaManager.getTableNames(); } else { - return brokerSegmentMetadataView.getDatasourceNames(); + return segmentMetadataCache.getDatasourceNames(); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 38491de49507..28e322a3ddca 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -133,7 +133,6 @@ public class SystemSchema extends AbstractSchema private static final long IS_ACTIVE_TRUE = 1L; private static final long IS_PUBLISHED_FALSE = 0L; private static final long IS_PUBLISHED_TRUE = 1L; - private static final long IS_AVAILABLE_TRUE = 1L; private static final long IS_OVERSHADOWED_FALSE = 0L; private static final long IS_OVERSHADOWED_TRUE = 1L; @@ -214,7 +213,6 @@ public class SystemSchema extends AbstractSchema @Inject public SystemSchema( - final DruidSchema druidSchema, final BrokerSegmentMetadataView metadataView, final TimelineServerView serverView, final FilteredServerInventoryView serverInventoryView, @@ -227,7 +225,7 @@ public SystemSchema( { Preconditions.checkNotNull(serverView, "serverView"); this.tableMap = ImmutableMap.of( - SEGMENTS_TABLE, new SegmentsTable(druidSchema, metadataView, jsonMapper, authorizerMapper), + SEGMENTS_TABLE, new SegmentsTable(metadataView, jsonMapper, authorizerMapper), SERVERS_TABLE, new ServersTable(druidNodeDiscoveryProvider, serverInventoryView, authorizerMapper, overlordClient, coordinatorDruidLeaderClient), SERVER_SEGMENTS_TABLE, new ServerSegmentsTable(serverView, authorizerMapper), TASKS_TABLE, new TasksTable(overlordClient, authorizerMapper), @@ -246,19 +244,16 @@ public Map getTableMap() */ static class SegmentsTable extends AbstractTable implements ScannableTable { - private final DruidSchema druidSchema; private final ObjectMapper jsonMapper; private final AuthorizerMapper authorizerMapper; private final BrokerSegmentMetadataView metadataView; public SegmentsTable( - DruidSchema druidSchemna, BrokerSegmentMetadataView metadataView, ObjectMapper jsonMapper, AuthorizerMapper authorizerMapper ) { - this.druidSchema = druidSchemna; this.metadataView = metadataView; this.jsonMapper = jsonMapper; this.authorizerMapper = authorizerMapper; @@ -299,7 +294,7 @@ public Enumerable scan(DataContext root) //is_active is true for published segments that are not overshadowed val.isOvershadowed() ? IS_ACTIVE_FALSE : IS_ACTIVE_TRUE, //is_published is true for published segments - val.isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, + (val.getIsRealtime() == 0) ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, val.getIsAvailable(), val.getIsRealtime(), val.isOvershadowed() ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE, @@ -378,7 +373,6 @@ protected static class SegmentTableView private final long numRows; private final Integer replicationFactor; private final boolean isOvershadowed; - private final boolean isPublished; public SegmentTableView( DataSegment segment, @@ -387,8 +381,7 @@ public SegmentTableView( long numReplicas, long numRows, Integer replicationFactor, - boolean isOvershadowed, - boolean isPublished + boolean isOvershadowed ) { this.segment = segment; @@ -398,7 +391,6 @@ public SegmentTableView( this.numRows = numRows; this.replicationFactor = replicationFactor; this.isOvershadowed = isOvershadowed; - this.isPublished = isPublished; } public DataSegment getSegment() @@ -436,9 +428,18 @@ public boolean isOvershadowed() return isOvershadowed; } - public boolean isPublished() + @Override + public String toString() { - return isPublished; + return "SegmentTableView{" + + "segmentId=" + segment.getId() + + ", isAvailable=" + isAvailable + + ", isRealtime=" + isRealtime + + ", numReplicas=" + numReplicas + + ", numRows=" + numRows + + ", replicationFactor=" + replicationFactor + + ", isOvershadowed=" + isOvershadowed + + '}'; } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java index ae5f14aab8c6..2cf08d7d6b71 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java @@ -344,7 +344,7 @@ public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedE ) { @Override - void markDataSourceAsNeedRebuild(String datasource) + public void markDataSourceAsNeedRebuild(String datasource) { super.markDataSourceAsNeedRebuild(datasource); markDataSourceLatch.countDown(); @@ -352,7 +352,7 @@ void markDataSourceAsNeedRebuild(String datasource) @Override @VisibleForTesting - void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException + public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { super.refresh(segmentsToRefresh, dataSourcesToRebuild); refreshLatch.countDown(); From 3aad095a8a547522a139e7fdd7babd40450dabf3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 6 Sep 2023 18:45:02 +0530 Subject: [PATCH 07/82] Fix dataSource schema on coordinator and minor renaming --- .../client/coordinator/CoordinatorClient.java | 6 ++-- .../coordinator/CoordinatorClientImpl.java | 14 +++++----- ...ourceSchema.java => DataSourceSchema.java} | 4 +-- .../metadata/SegmentMetadataCache.java | 16 +++++------ .../metadata/SegmentMetadataCacheConfig.java | 8 +++++- .../druid/server/http/MetadataResource.java | 28 ++++++++++++------- .../org/apache/druid/cli/CliCoordinator.java | 2 +- .../schema/BrokerSegmentMetadataCache.java | 6 ++-- .../BrokerSegmentMetadataCacheConfig.java | 8 ++++++ 9 files changed, 56 insertions(+), 36 deletions(-) rename server/src/main/java/org/apache/druid/segment/metadata/{DatasourceSchema.java => DataSourceSchema.java} (92%) diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index a56363afee83..ad72159bde02 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -20,14 +20,12 @@ package org.apache.druid.client.coordinator; import com.google.common.util.concurrent.ListenableFuture; -import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentStatusInCluster; import org.joda.time.Interval; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -48,7 +46,7 @@ public interface CoordinatorClient */ ListenableFuture> fetchUsedSegments(String dataSource, List intervals); - ListenableFuture> fetchDatasourceSchema(Set datasources); + ListenableFuture> fetchDataSourceSchema(Set datasources); /** * Returns a new instance backed by a ServiceClient which follows the provided retryPolicy diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index 2a9e1891f086..24e6968ad6ab 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -30,7 +30,7 @@ import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; import org.apache.druid.rpc.ServiceRetryPolicy; -import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.timeline.DataSegment; import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; @@ -110,20 +110,20 @@ public ListenableFuture> fetchUsedSegments(String dataSource, } @Override - public ListenableFuture> fetchDatasourceSchema(Set datasources) + public ListenableFuture> fetchDataSourceSchema(Set dataSources) { - final String path = "/druid/coordinator/v1/metadata/datasourceSchema"; - if (null == datasources) + final String path = "/druid/coordinator/v1/metadata/dataSourceSchema"; + if (null == dataSources) { - datasources = new HashSet<>(); + dataSources = new HashSet<>(); } return FutureUtils.transform( client.asyncRequest( new RequestBuilder(HttpMethod.POST, path) - .jsonContent(jsonMapper, datasources), + .jsonContent(jsonMapper, dataSources), new BytesFullResponseHandler() ), - holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) + holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) ); } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java similarity index 92% rename from server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java rename to server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java index 9fb13fe5f0e5..15e87cf5d072 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java @@ -4,14 +4,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.segment.column.RowSignature; -public class DatasourceSchema +public class DataSourceSchema { // dsinfo private final String datasource; private final RowSignature rowSignature; @JsonCreator - public DatasourceSchema( + public DataSourceSchema( @JsonProperty("datasource") String datasource, @JsonProperty("rowSignature") RowSignature rowSignature) { diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index da32a86cdc2a..3c20b4bcd779 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -124,7 +124,7 @@ public class SegmentMetadataCache * Map of DataSource -> DruidTable. * This map can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final ConcurrentMap tables = new ConcurrentHashMap<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -181,7 +181,7 @@ public class SegmentMetadataCache * Currently, there are 2 threads that can access these variables. * * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DatasourceSchema} if necessary + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceSchema} if necessary * based on the information collected via timeline callbacks. */ private final Object lock = new Object(); @@ -429,13 +429,13 @@ public void refresh(final Set segmentsToRefresh, final Set da public void rebuildDatasource(String dataSource) { - final DatasourceSchema druidTable = buildDruidTable(dataSource); + final DataSourceSchema druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final DatasourceSchema oldTable = tables.put(dataSource, druidTable); + final DataSourceSchema oldTable = tables.put(dataSource, druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -455,12 +455,12 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } - public DatasourceSchema getDatasource(String name) + public DataSourceSchema getDatasource(String name) { return tables.get(name); } - public Map getDatasourceSchemaMap() + public Map getDataSourceSchemaMap() { return ImmutableMap.copyOf(tables); } @@ -819,7 +819,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin @VisibleForTesting @Nullable - public DatasourceSchema buildDruidTable(final String dataSource) + public DataSourceSchema buildDruidTable(final String dataSource) { ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); @@ -847,7 +847,7 @@ public DatasourceSchema buildDruidTable(final String dataSource) final RowSignature.Builder builder = RowSignature.builder(); columnTypes.forEach(builder::add); - return new DatasourceSchema(dataSource, builder.build()); + return new DataSourceSchema(dataSource, builder.build()); } public Map getSegmentMetadataSnapshot() diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index c5d22c087794..80aba42ecddc 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -6,9 +6,10 @@ public class SegmentMetadataCacheConfig { @JsonProperty - private boolean awaitInitializationOnStart = true; + private boolean awaitInitializationOnStart = false; @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); + private boolean enabled = false; @JsonProperty private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); @@ -41,4 +42,9 @@ public Period getMetadataRefreshPeriod() { return metadataRefreshPeriod; } + + public boolean isEnabled() + { + return enabled; + } } diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 69dd113e180b..5b70be95b5b4 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -29,11 +29,10 @@ import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.Segments; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.segment.metadata.SegmentMetadataCache; -import org.apache.druid.metadata.SegmentsMetadataManager; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.JettyUtils; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.http.security.DatasourceResourceFilter; @@ -44,7 +43,6 @@ import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.joda.time.Interval; -import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; @@ -58,6 +56,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -361,17 +360,26 @@ public Response getUsedSegment( } @POST - @Path("/datasourceSchema") - public Response getDatasourceSchema( - List datasources + @Path("/dataSourceSchema") + @Produces(MediaType.APPLICATION_JSON) + public Response getDataSourceSchema( + List dataSources ) { if (null == segmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } - Map datasourceSchemaMap = segmentMetadataCache.getDatasourceSchemaMap(); - datasourceSchemaMap.keySet().retainAll(datasources); + Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceSchemaMap(); + + List results = new ArrayList<>(); + List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; + + for (Map.Entry entry : dataSourceSchemaMap.entrySet()) { + if (dataSourcesToRetain.contains(entry.getKey())) { + results.add(entry.getValue()); + } + } - return Response.status(Response.Status.OK).entity(datasourceSchemaMap.values()).build(); + return Response.status(Response.Status.OK).entity(results).build(); } } diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 6c16684a2e93..c9b23df64fb2 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -220,6 +220,7 @@ public void configure(Binder binder) "druid.coordinator.balancer.cachingCost", CachingCostBalancerStrategyConfig.class ); + JsonConfigProvider.bind(binder, "druid.coordinator.segmentMetadataCache", SegmentMetadataCacheConfig.class); binder.bind(RedirectFilter.class).in(LazySingleton.class); if (beOverlord) { @@ -532,7 +533,6 @@ public void configure(Binder binder) binder.install(new BrokerProcessingModule()); binder.install(new JoinableFactoryModule()); - JsonConfigProvider.bind(binder, "druid.coordinator.segment", SegmentMetadataCacheConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index a7f47d6c5fde..8f3cd67abe66 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -9,7 +9,7 @@ import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; @@ -73,7 +73,7 @@ public void refresh(final Set segmentsToRefresh, final Set da // Fetch dataSource schema from the Coordinator try { - FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema(dataSourcesToQuery), true) + FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceSchema(dataSourcesToQuery), true) .forEach(item -> polledDataSourceSchema.put( item.getDatasource(), physicalDatasourceMetadataBuilder.build( @@ -121,7 +121,7 @@ public void refresh(final Set segmentsToRefresh, final Set da @Override public void rebuildDatasource(String dataSource) { - final DatasourceSchema druidTable = buildDruidTable(dataSource); + final DataSourceSchema druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 7f4041fb0b03..00a068a79950 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -11,6 +11,9 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig @JsonProperty private long metadataSegmentPollPeriod = 60000; + @JsonProperty + private boolean awaitInitializationOnStart = true; + public boolean isMetadataSegmentCacheEnable() { return metadataSegmentCacheEnable; @@ -21,4 +24,9 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } + @Override + public boolean isAwaitInitializationOnStart() + { + return awaitInitializationOnStart; + } } From 14baf19cb2bd15472e5cf2e10986d2c63a03e6b2 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 7 Sep 2023 13:47:55 +0530 Subject: [PATCH 08/82] cleanup and fix some tests --- .../timeline/SegmentStatusInCluster.java | 2 + .../timeline/SegmentStatusInClusterTest.java | 2 +- .../apache/druid/client/BrokerServerView.java | 13 +--- .../druid/client/CoordinatorServerView.java | 2 +- ...toryView.java => CoordinatorTimeline.java} | 14 +++- .../druid/client/InternalQueryConfig.java | 4 +- ...va => QueryableCoordinatorServerView.java} | 4 +- .../client/coordinator/CoordinatorClient.java | 5 +- .../metadata/AvailableSegmentMetadata.java | 12 +--- .../segment/metadata/DataSourceSchema.java | 37 +++++++++- .../server/http/DataSourcesResource.java | 8 +-- .../druid/server/http/MetadataResource.java | 33 ++++----- ...st.java => CoordinatorServerViewTest.java} | 8 +-- .../SegmentDataCacheConcurrencyTest.java | 4 +- .../metadata}/SegmentMetadataCacheCommon.java | 5 +- .../metadata}/SegmentMetadataCacheTest.java | 71 ++++++++----------- .../CuratorDruidCoordinatorTest.java | 6 +- .../server/http/DataSourcesResourceTest.java | 14 ++-- .../server/http/MetadataResourceTest.java | 13 ++-- .../server/http/ServersResourceTest.java | 4 +- .../org/apache/druid/cli/CliCoordinator.java | 14 ++-- .../schema/BrokerSegmentMetadataCache.java | 19 +++++ .../BrokerSegmentMetadataCacheConfig.java | 19 +++++ .../schema/BrokerSegmentMetadataView.java | 3 +- .../schema/DruidCalciteSchemaModule.java | 18 +---- .../PhysicalDatasourceMetadataBuilder.java | 19 +++++ .../org/apache/druid/sql/guice/SqlModule.java | 2 +- .../sql/calcite/schema/SystemSchemaTest.java | 24 ++++--- 28 files changed, 216 insertions(+), 163 deletions(-) rename server/src/main/java/org/apache/druid/client/{CoordinatorInventoryView.java => CoordinatorTimeline.java} (59%) rename server/src/main/java/org/apache/druid/client/{TimelineAwareCoordinatorServerView.java => QueryableCoordinatorServerView.java} (96%) rename server/src/test/java/org/apache/druid/client/{TimelineAwareCoordinatorServerViewTest.java => CoordinatorServerViewTest.java} (98%) rename {sql/src/test/java/org/apache/druid/sql/calcite/schema => server/src/test/java/org/apache/druid/segment/metadata}/SegmentDataCacheConcurrencyTest.java (99%) rename {sql/src/test/java/org/apache/druid/sql/calcite/schema => server/src/test/java/org/apache/druid/segment/metadata}/SegmentMetadataCacheCommon.java (94%) rename {sql/src/test/java/org/apache/druid/sql/calcite/schema => server/src/test/java/org/apache/druid/segment/metadata}/SegmentMetadataCacheTest.java (96%) diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index bccfa764726b..a440a36923e6 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -32,6 +32,8 @@ *
  • the {@code DataSegment} object
    • *
  • overshadowed status of the segment
    • *
  • replication factor of the segment
    • + *
  • number of rows in the segment
    • + *
  • if the segment is realtime
    • * *

    * Objects of this class are used to sync the state of segments from the Coordinator to different services, typically the Broker. diff --git a/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java b/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java index 6e604f7ae9da..54190870e77c 100644 --- a/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java +++ b/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java @@ -76,7 +76,7 @@ private static SegmentStatusInCluster createSegmentForTest() 1 ); - return new SegmentStatusInCluster(dataSegment, OVERSHADOWED, REPLICATION_FACTOR, 1L, 0L, true); + return new SegmentStatusInCluster(dataSegment, OVERSHADOWED, REPLICATION_FACTOR, 10L, true); } @Test diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index 11624aad752a..efaf09232c35 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import com.google.inject.Inject; import org.apache.druid.client.selector.QueryableDruidServer; @@ -433,12 +434,7 @@ public List getDruidServers() public Map getSegmentMetadata() { - return selectors; - } - - public Set getDatasourceNames() - { - return timelines.keySet(); + return ImmutableMap.copyOf(selectors); } Object getLock() @@ -455,9 +451,4 @@ Map getSelectors() { return selectors; } - - ConcurrentMap getClients() - { - return clients; - } } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index be4e90d9e600..3955fdbf7f1a 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -26,7 +26,7 @@ * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class CoordinatorServerView implements CoordinatorInventoryView +public class CoordinatorServerView implements CoordinatorTimeline { private static final Logger log = new Logger(CoordinatorServerView.class); diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java b/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java similarity index 59% rename from server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java rename to server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java index a938c8864f1b..0998d5f79db3 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java @@ -6,8 +6,20 @@ import java.util.Map; -public interface CoordinatorInventoryView extends InventoryView +/** + * Segment timeline maintained in the coordinator. + */ +public interface CoordinatorTimeline extends InventoryView { + /** + * Retrieve timeline for a dataSource. + */ VersionedIntervalTimeline getTimeline(DataSource dataSource); + + /** + * Server information for all segments in the timeline. + */ Map getSegmentLoadInfos(); + + } diff --git a/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java b/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java index 3c1d4d96022b..1098103ffe5c 100644 --- a/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java +++ b/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java @@ -26,8 +26,8 @@ /** * This class contains configuration that internally generated Druid queries - * should add to their query payload. The runtime properties for this class - * have the prefix "druid.broker.internal.query.config." + * should add to their query payload. The runtime properties for this class have + * the prefix "druid.{service}.internal.query.config.". */ public class InternalQueryConfig { diff --git a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java similarity index 96% rename from server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java rename to server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 88661386f0bc..ade760710def 100644 --- a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -45,12 +45,12 @@ * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class TimelineAwareCoordinatorServerView extends BrokerServerView implements CoordinatorInventoryView +public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline { private final FilteredServerInventoryView baseView; @Inject - public TimelineAwareCoordinatorServerView( + public QueryableCoordinatorServerView( final QueryToolChestWarehouse warehouse, final QueryWatcher queryWatcher, final @Smile ObjectMapper smileMapper, diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index ad72159bde02..66b739cf51cb 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -20,9 +20,9 @@ package org.apache.druid.client.coordinator; import com.google.common.util.concurrent.ListenableFuture; -import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; @@ -46,6 +46,9 @@ public interface CoordinatorClient */ ListenableFuture> fetchUsedSegments(String dataSource, List intervals); + /** + * Fetches schema for the given dataSources. + */ ListenableFuture> fetchDataSourceSchema(Set datasources); /** diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java index cd2f849438ee..a074f8e2bc9d 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java @@ -27,7 +27,7 @@ import java.util.Set; /** - * Immutable representation of RowSignature and other segment attributes needed by {@code SystemSchema.SegmentsTable} + * Immutable representation of RowSignature and other segment attributes. * This class contains the metadata of segments announced by historicals or ingestion tasks. */ public class AvailableSegmentMetadata @@ -158,14 +158,4 @@ public AvailableSegmentMetadata build() return new AvailableSegmentMetadata(this); } } - - @Override - public String toString() - { - return "AvailableSegmentMetadata{" + - "segmentId=" + segment.getId() + - ", isRealtime=" + isRealtime + - ", numRows=" + numRows + - '}'; - } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java index 15e87cf5d072..0528cfe3b1f8 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java @@ -4,9 +4,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.segment.column.RowSignature; +import java.util.Objects; + +/** + * Encapsulates schema information of a dataSource. + */ public class DataSourceSchema { - // dsinfo private final String datasource; private final RowSignature rowSignature; @@ -30,4 +34,35 @@ public RowSignature getRowSignature() { return rowSignature; } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DataSourceSchema that = (DataSourceSchema) o; + return Objects.equals(datasource, that.datasource) && Objects.equals( + rowSignature, + that.rowSignature + ); + } + + @Override + public int hashCode() + { + return Objects.hash(datasource, rowSignature); + } + + @Override + public String toString() + { + return "DataSourceSchema{" + + "datasource='" + datasource + '\'' + + ", rowSignature=" + rowSignature + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index ea92c4e8471a..1cbaa92ae231 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -32,7 +32,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; -import org.apache.druid.client.CoordinatorInventoryView; +import org.apache.druid.client.CoordinatorTimeline; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableSegmentLoadInfo; import org.apache.druid.client.SegmentLoadInfo; @@ -102,7 +102,7 @@ public class DataSourcesResource private static final Logger log = new Logger(DataSourcesResource.class); private static final long DEFAULT_LOADSTATUS_INTERVAL_OFFSET = 14 * 24 * 60 * 60 * 1000; - private final CoordinatorInventoryView serverInventoryView; + private final CoordinatorTimeline serverInventoryView; private final SegmentsMetadataManager segmentsMetadataManager; private final MetadataRuleManager metadataRuleManager; private final OverlordClient overlordClient; @@ -111,7 +111,7 @@ public class DataSourcesResource @Inject public DataSourcesResource( - CoordinatorInventoryView serverInventoryView, + CoordinatorTimeline coordinatorTimeline, SegmentsMetadataManager segmentsMetadataManager, MetadataRuleManager metadataRuleManager, @Nullable OverlordClient overlordClient, @@ -119,7 +119,7 @@ public DataSourcesResource( DruidCoordinator coordinator ) { - this.serverInventoryView = serverInventoryView; + this.serverInventoryView = coordinatorTimeline; this.segmentsMetadataManager = segmentsMetadataManager; this.metadataRuleManager = metadataRuleManager; this.overlordClient = overlordClient; diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 5b70be95b5b4..1de89a11de73 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -214,8 +214,8 @@ private Response getAllUsedSegmentsWithAdditionalDetails( Integer replicationFactor = isOvershadowed ? (Integer) 0 : coordinator.getReplicationFactor(segment.getId()); segmentAlreadySeen.add(segment.getId()); - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, 20L, true); - }).peek(v -> log.info("peeking into first stream segmentseen [%s], id [%s]", segmentAlreadySeen, v.getDataSegment().getId())); + return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, numRows, true); + }); Stream finalSegments = segmentStatus; @@ -225,29 +225,17 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .getSegmentMetadataSnapshot() .values() .stream() - .peek(v -> log.info( - "peeking into second stream (first) segmentseen [%s], id [%s]", - segmentAlreadySeen, - v.getSegment().getId() - )) .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment() .getId())) - .map(availableSegmentMetadata -> { - return new SegmentStatusInCluster( - availableSegmentMetadata.getSegment(), - false, - (int) availableSegmentMetadata.getNumReplicas(), - /**availableSegmentMetadata.getNumRows(), **/30L, - availableSegmentMetadata.isRealtime() != 0 - ); - }) - .peek(v -> log.info("peeking into second stream (second) segmentseen [%s], id [%s]", - segmentAlreadySeen, - v.getDataSegment().getId() + .map(availableSegmentMetadata -> new SegmentStatusInCluster( + availableSegmentMetadata.getSegment(), + false, + (int) availableSegmentMetadata.getNumReplicas(), + availableSegmentMetadata.getNumRows(), + availableSegmentMetadata.isRealtime() != 0 )); - finalSegments = Stream.concat(segmentStatus, realtimeSegmentStatus) - .peek(v -> log.info("combined stream element %s", v)); + finalSegments = Stream.concat(segmentStatus, realtimeSegmentStatus); } final Function> raGenerator = segment -> Collections @@ -359,6 +347,9 @@ public Response getUsedSegment( return Response.status(Response.Status.NOT_FOUND).build(); } + /** + * Return schema for the given dataSources. + */ @POST @Path("/dataSourceSchema") @Produces(MediaType.APPLICATION_JSON) diff --git a/server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java similarity index 98% rename from server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java index 190a41e53f9e..1bca2a0690af 100644 --- a/server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java @@ -52,7 +52,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class TimelineAwareCoordinatorServerViewTest extends CuratorTestBase +public class CoordinatorServerViewTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; @@ -63,9 +63,9 @@ public class TimelineAwareCoordinatorServerViewTest extends CuratorTestBase private CountDownLatch segmentRemovedLatch; private BatchServerInventoryView baseView; - private TimelineAwareCoordinatorServerView overlordServerView; + private CoordinatorServerView overlordServerView; - public TimelineAwareCoordinatorServerViewTest() + public CoordinatorServerViewTest() { jsonMapper = TestHelper.makeJsonMapper(); zkPathsConfig = new ZkPathsConfig(); @@ -332,7 +332,7 @@ public CallbackAction segmentViewInitialized() } }; - overlordServerView = new TimelineAwareCoordinatorServerView( + overlordServerView = new CoordinatorServerView( baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter() diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java similarity index 99% rename from sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index d34bed3ccf73..dbc104eafa2b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; @@ -42,9 +42,7 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.metadata.PhysicalDatasourceMetadata; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.TableDataSource; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java similarity index 94% rename from sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java rename to server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 97534f2f81d9..acf2c6b76c9d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -33,9 +33,6 @@ import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.util.CalciteTestBase; -import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.easymock.EasyMock; import org.junit.AfterClass; import org.junit.Before; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java similarity index 96% rename from sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 2cf08d7d6b71..5f1fccbd15f3 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -26,10 +26,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.data.input.InputRow; @@ -40,8 +36,6 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; @@ -72,13 +66,6 @@ import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.table.DatasourceTable; -import org.apache.druid.sql.calcite.table.DruidTable; -import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; -import org.apache.druid.sql.calcite.util.TestDataBuilder; -import org.apache.druid.sql.calcite.util.TestServerInventoryView; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.DataSegment.PruneSpecsHolder; import org.apache.druid.timeline.SegmentId; @@ -133,7 +120,7 @@ public void setUp() throws Exception .withRollup(false) .build() ) - .rows(ROWS1) + .rows(SegmentMetadataCacheCommon.ROWS1) .buildMMappedIndex(); final QueryableIndex index2 = IndexBuilder.create() @@ -145,7 +132,7 @@ public void setUp() throws Exception .withRollup(false) .build() ) - .rows(ROWS2) + .rows(SegmentMetadataCacheCommon.ROWS2) .buildMMappedIndex(); final InputRowSchema rowSchema = new InputRowSchema( @@ -219,7 +206,7 @@ public void setUp() throws Exception .rows(autoRows2) .buildMMappedIndex(); - walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( + walker = new SpecificSegmentsQuerySegmentWalker(SegmentMetadataCacheCommon.conglomerate).add( DataSegment.builder() .dataSource(CalciteTests.DATASOURCE1) .interval(Intervals.of("2000/P1Y")) @@ -285,14 +272,14 @@ public void setUp() throws Exception public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { - return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); + return buildSchemaMarkAndTableLatch(SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT); } public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException { Preconditions.checkState(runningSchema == null); runningSchema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory( @@ -330,14 +317,14 @@ public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedE { Preconditions.checkState(runningSchema == null); runningSchema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory( ImmutableSet.of(globalTableJoinable), ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) ), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -704,11 +691,11 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -747,11 +734,11 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -794,11 +781,11 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -838,11 +825,11 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -879,11 +866,11 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -937,11 +924,11 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -998,11 +985,11 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1033,11 +1020,11 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1081,11 +1068,11 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1327,7 +1314,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception ImmutableSet.of(globalTableJoinable), ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) ), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), internalQueryConfig, new NoopServiceEmitter() @@ -1459,11 +1446,11 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), emitter diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index 647eab9c6c45..b9b6fc8516fe 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -30,7 +30,7 @@ import org.apache.curator.utils.ZKPaths; import org.apache.druid.client.BatchServerInventoryView; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; -import org.apache.druid.client.Alpha; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -97,7 +97,7 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase private static final long COORDINATOR_PERIOD = 100; private BatchServerInventoryView baseView; - private Alpha serverView; + private CoordinatorServerView serverView; private CountDownLatch segmentViewInitLatch; /** * The following two fields are changed during {@link #testMoveSegment()}, the change might not be visible from the @@ -405,7 +405,7 @@ public CallbackAction segmentViewInitialized() } }; - serverView = new Alpha(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); + serverView = new CoordinatorServerView(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); baseView.start(); diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 5cdf1d5b73b9..551151458546 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -28,7 +28,7 @@ import com.google.common.util.concurrent.Futures; import it.unimi.dsi.fastutil.objects.Object2LongMap; import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; -import org.apache.druid.client.Alpha; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -83,7 +83,7 @@ public class DataSourcesResourceTest { - private Alpha inventoryView; + private CoordinatorServerView inventoryView; private DruidServer server; private List listDataSources; private List dataSegmentList; @@ -94,7 +94,7 @@ public class DataSourcesResourceTest public void setUp() { request = EasyMock.createStrictMock(HttpServletRequest.class); - inventoryView = EasyMock.createStrictMock(Alpha.class); + inventoryView = EasyMock.createStrictMock(CoordinatorServerView.class); server = EasyMock.niceMock(DruidServer.class); dataSegmentList = new ArrayList<>(); dataSegmentList.add( @@ -1302,7 +1302,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1318,7 +1318,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1381,7 +1381,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1397,7 +1397,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index 717db5f03caa..c0978e80c367 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -101,22 +101,23 @@ public void setUp() segmentsMetadataManager, storageCoordinator, AuthTestUtils.TEST_AUTHORIZER_MAPPER, - coordinator + coordinator, + null ); } @Test public void testGetAllSegmentsWithOvershadowedStatus() { - Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus"); + Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", null); final List resultList = extractSegmentStatusList(response); Assert.assertEquals(resultList.size(), 4); - Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 5L, 0L, true), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 5L, 0L, true), resultList.get(1)); - Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, 5L, 0L, true), resultList.get(2)); + Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 5L, true), resultList.get(0)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 5L, true), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, 5L, true), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, 5L, 0L, true), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, 5L, true), resultList.get(3)); } @Test diff --git a/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java b/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java index 951070557d7d..f96d2a3e74c8 100644 --- a/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import org.apache.druid.client.Alpha; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidServer; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; @@ -53,7 +53,7 @@ public void setUp() .build(); dummyServer.addDataSegment(segment); - Alpha inventoryView = EasyMock.createMock(Alpha.class); + CoordinatorServerView inventoryView = EasyMock.createMock(CoordinatorServerView.class); EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(dummyServer)).anyTimes(); EasyMock.expect(inventoryView.getInventoryValue(dummyServer.getName())).andReturn(dummyServer).anyTimes(); EasyMock.replay(inventoryView); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index c9b23df64fb2..652673e4239f 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -41,9 +41,9 @@ import org.apache.druid.client.CoordinatorSegmentWatcherConfig; import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.HttpServerInventoryViewResource; -import org.apache.druid.client.CoordinatorInventoryView; +import org.apache.druid.client.CoordinatorTimeline; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.client.TimelineAwareCoordinatorServerView; +import org.apache.druid.client.QueryableCoordinatorServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; @@ -232,7 +232,7 @@ public void configure(Binder binder) if (isSegmentMetadataCacheEnabled()) { binder.install(new SegmentMetadataCacheModule()); } else { - binder.bind(CoordinatorInventoryView.class).to(CoordinatorServerView.class).in(LazySingleton.class); + binder.bind(CoordinatorTimeline.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); binder.bind(SegmentMetadataCache.class).toProvider(Providers.of(null)); } @@ -541,10 +541,10 @@ public void configure(Binder binder) binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); binder.bind(CachingClusteredClient.class).in(LazySingleton.class); - binder.bind(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); - binder.bind(CoordinatorInventoryView.class).to(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); - binder.bind(TimelineServerView.class).to(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); - LifecycleModule.register(binder, TimelineAwareCoordinatorServerView.class); + binder.bind(QueryableCoordinatorServerView.class).in(LazySingleton.class); + binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); + binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); + LifecycleModule.register(binder, QueryableCoordinatorServerView.class); LifecycleModule.register(binder, SegmentMetadataCache.class); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 8f3cd67abe66..b55dc855572a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.sql.calcite.schema; import com.google.common.collect.Sets; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 00a068a79950..834b2d7079e1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.sql.calcite.schema; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index 9eeafcd4f8c4..fc20c38d00a4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -61,7 +61,7 @@ /** * This class polls the Coordinator in background to keep the latest published segments. - * Provides {@link #getSegmentMetadata()} for others to get segments in metadata store. + * Provides {@link #getSegmentTableView()} for sys segments table view. * * This class polls the data from {@link SegmentsMetadataManager} object in the memory of the * currently leading Coordinator via HTTP queries. @@ -284,6 +284,7 @@ private ImmutableSortedSet fetchSegmentMetadata() return builder.build(); } + // Note that coordinator must be up to get segments private JsonParserIterator querySegmentMetadata( Set watchedDataSources diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index 3003b99fbe1c..132f2cb92590 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -38,16 +38,8 @@ public class DruidCalciteSchemaModule implements Module { private static final String DRUID_SCHEMA_NAME = "druid"; private static final String INFORMATION_SCHEMA_NAME = "INFORMATION_SCHEMA"; - private static final String SEGMENT_METADATA_CACHE_ENABLED = "druid.sql.planner.segmentMetadataCacheEnabled"; static final String INCOMPLETE_SCHEMA = "INCOMPLETE_SCHEMA"; - private final Properties properties; - - public DruidCalciteSchemaModule(Properties properties) - { - this.properties = properties; - } - @Override public void configure(Binder binder) { @@ -60,9 +52,8 @@ public void configure(Binder binder) .in(Scopes.SINGLETON); // BrokerSegmentMetadataCache needs to listen to changes for incoming segments - if (isSegmentMetadataCacheEnabled()) { - LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); - } + LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); + binder.bind(DruidSchema.class).in(Scopes.SINGLETON); binder.bind(SystemSchema.class).in(Scopes.SINGLETON); binder.bind(InformationSchema.class).in(Scopes.SINGLETON); @@ -82,9 +73,4 @@ private DruidSchemaCatalog getRootSchema(@Named(INCOMPLETE_SCHEMA) DruidSchemaCa rootSchema.getRootSchema().add(INFORMATION_SCHEMA_NAME, informationSchema); return rootSchema; } - - private boolean isSegmentMetadataCacheEnabled() - { - return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED, "true")); - } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index c9760aeb0cf4..5dfbfb664440 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.sql.calcite.schema; import com.google.inject.Inject; diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 03a774a05b61..56d0d2d5d41f 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -104,7 +104,7 @@ public void configure(Binder binder) binder.bind(TableDefnRegistry.class).in(LazySingleton.class); - binder.install(new DruidCalciteSchemaModule(props)); + binder.install(new DruidCalciteSchemaModule()); binder.install(new CalcitePlannerModule()); binder.install(new SqlAggregationModule()); binder.install(new DruidViewModule()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 52439c33d6e9..659be26029e4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -42,6 +42,7 @@ import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.InputRow; import org.apache.druid.discovery.DataNodeService; @@ -252,15 +253,17 @@ public void setUp() throws Exception .add(segment2, index2) .add(segment3, index3); - SegmentMetadataCache cache = new SegmentMetadataCache( + BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), new TestServerInventoryView(walker.getSegments(), realtimeSegments), - new SegmentManager(EasyMock.createMock(SegmentLoader.class)), - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), - new NoopServiceEmitter() + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder( + new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), + new SegmentManager(EasyMock.createMock(SegmentLoader.class))), + new NoopCoordinatorClient() ); cache.start(); cache.awaitInitialization(); @@ -269,7 +272,6 @@ public void setUp() throws Exception druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class); schema = new SystemSchema( - druidSchema, metadataView, serverView, serverInventoryView, @@ -566,13 +568,13 @@ public void testGetTableMap() @Test public void testSegmentsTable() throws Exception { - final SegmentsTable segmentsTable = new SegmentsTable(druidSchema, metadataView, new ObjectMapper(), authMapper); + final SegmentsTable segmentsTable = new SegmentsTable(metadataView, new ObjectMapper(), authMapper); final Set publishedSegments = new HashSet<>(Arrays.asList( - new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 2L, 0L, true), - new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 2L, 0L, true), - new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 2L, 0L, true), - new SegmentStatusInCluster(segment1, true, 2, 2L, 0L, true), - new SegmentStatusInCluster(segment2, false, 0, 2L, 0L, true) + new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 2L, true), + new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 2L, true), + new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 2L, true), + new SegmentStatusInCluster(segment1, true, 2, 2L, true), + new SegmentStatusInCluster(segment2, false, 0, 2L, true) )); EasyMock.expect(metadataView.getSegmentMetadata()).andReturn(publishedSegments.iterator()).once(); From 857f056a130736112b0a14d12d7dd07da18397ba Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 7 Sep 2023 20:31:37 +0530 Subject: [PATCH 09/82] Port tests and test build failure --- ...ruidSchemaInternRowSignatureBenchmark.java | 8 - .../druid/benchmark/query/SqlBenchmark.java | 2 +- .../query/SqlExpressionBenchmark.java | 2 +- .../query/SqlNestedDataBenchmark.java | 2 +- .../benchmark/query/SqlVsNativeBenchmark.java | 2 +- ...ressedBigDecimalSqlAggregatorTestBase.java | 2 +- .../sql/TDigestSketchSqlAggregatorTest.java | 2 +- .../hll/sql/HllSketchSqlAggregatorTest.java | 2 +- .../sql/DoublesSketchSqlAggregatorTest.java | 2 +- .../sql/ThetaSketchSqlAggregatorTest.java | 2 +- ...ArrayOfDoublesSketchSqlAggregatorTest.java | 2 +- .../sql/BloomFilterSqlAggregatorTest.java | 2 +- ...etsHistogramQuantileSqlAggregatorTest.java | 2 +- .../sql/QuantileSqlAggregatorTest.java | 2 +- .../apache/druid/msq/test/MSQTestBase.java | 2 +- .../msq/test/MSQTestControllerContext.java | 2 +- .../sql/VarianceSqlAggregatorTest.java | 2 +- .../apache/druid/client/BrokerServerView.java | 31 +-- .../QueryableCoordinatorServerView.java | 6 +- .../metadata/SegmentMetadataCache.java | 43 ++- .../metadata/SegmentMetadataCacheConfig.java | 4 +- .../coordinator/NoopCoordinatorClient.java | 8 + .../SegmentDataCacheConcurrencyTest.java | 20 +- .../metadata/SegmentMetadataCacheCommon.java | 126 +++++---- .../SegmentMetadataCacheConfigTest.java | 23 +- .../metadata/SegmentMetadataCacheTest.java | 246 +++--------------- .../metadata/TestTimelineServerView.java | 8 +- .../SpecificSegmentsQuerySegmentWalker.java | 5 +- .../schema/BrokerSegmentMetadataCache.java | 12 +- .../BrokerSegmentMetadataCacheConfig.java | 5 + .../schema/BrokerSegmentMetadataView.java | 11 +- .../apache/druid/sql/SqlStatementTest.java | 2 +- .../sql/avatica/DruidAvaticaHandlerTest.java | 2 +- .../druid/sql/avatica/DruidStatementTest.java | 2 +- .../sql/calcite/BaseCalciteQueryTest.java | 2 +- .../calcite/CalciteNestedDataQueryTest.java | 2 +- .../sql/calcite/DrillWindowQueryTest.java | 2 +- .../SqlVectorizedExpressionSanityTest.java | 2 +- .../schema/DruidSchemaNoDataInitTest.java | 19 +- .../sql/calcite/schema/SystemSchemaTest.java | 29 ++- .../druid/sql/calcite/util/CalciteTests.java | 37 ++- .../sql/calcite/util/QueryFrameworkUtils.java | 42 +-- .../sql/calcite/util/SqlTestFramework.java | 1 + .../sql/calcite/util/TestDataBuilder.java | 1 + .../druid/sql/http/SqlResourceTest.java | 2 +- 45 files changed, 288 insertions(+), 445 deletions(-) rename {sql/src/test/java/org/apache/druid/sql/calcite/planner => server/src/test/java/org/apache/druid/segment/metadata}/SegmentMetadataCacheConfigTest.java (75%) rename sql/src/test/java/org/apache/druid/sql/calcite/util/TestServerInventoryView.java => server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java (96%) rename {sql/src/test/java/org/apache/druid/sql/calcite/util => server/src/test/java/org/apache/druid/server}/SpecificSegmentsQuerySegmentWalker.java (97%) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index e87efb348079..4d2d11ba1f3f 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -76,9 +76,6 @@ private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCac public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, final TimelineServerView serverView, - final SegmentManager segmentManager, - final JoinableFactory joinableFactory, - final PlannerConfig config, final Escalator escalator, final InternalQueryConfig internalQueryConfig ) @@ -86,8 +83,6 @@ public SegmentMetadataCacheForBenchmark( super( queryLifecycleFactory, serverView, - segmentManager, - joinableFactory, SegmentMetadataCacheConfig.create(), escalator, internalQueryConfig, @@ -177,9 +172,6 @@ public void setup() EasyMock.mock(QueryLifecycleFactory.class), EasyMock.mock(TimelineServerView.class), null, - null, - EasyMock.mock(PlannerConfig.class), - null, null ); DruidServerMetadata serverMetadata = new DruidServerMetadata( diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index 1172d823e024..d340f10e35e3 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -63,7 +63,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 6f96a2cde568..09b5a74d4380 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -48,7 +48,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java index fe5dfff0c846..138de4ea3132 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java @@ -57,7 +57,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java index de3db00accf5..8c709a7457f9 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java @@ -49,7 +49,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java index 21dc9d9f0323..b6aa0d866e5d 100644 --- a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java +++ b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java @@ -44,7 +44,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java index f42cade76737..dd15a6defac2 100644 --- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java @@ -49,7 +49,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java index 349f1a57d1c0..c08bc8f04345 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java @@ -76,7 +76,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.DataSegment; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java index c2d81cede8ce..68f53687f892 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java @@ -60,7 +60,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java index 3946ce558b19..b4d7e29661cb 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java @@ -62,7 +62,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.DataSegment; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java index a240f89bdcc8..f262accfef94 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java @@ -49,7 +49,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java index 1c77f0986e11..30b76e8602b5 100644 --- a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java @@ -50,7 +50,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java index d0a12ab2f2d5..fea3d871e938 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java @@ -52,7 +52,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index 9e36def53cb8..0c81b4c40fcb 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -51,7 +51,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java index 4e00fd657ac5..ec97a34dfff0 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java @@ -162,7 +162,7 @@ import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.LookylooModule; import org.apache.druid.sql.calcite.util.QueryFrameworkUtils; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.view.InProcessViewManager; import org.apache.druid.sql.guice.SqlBindings; diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestControllerContext.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestControllerContext.java index 2ee2207fd83a..c1a33a4665e4 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestControllerContext.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestControllerContext.java @@ -48,7 +48,7 @@ import org.apache.druid.msq.util.MultiStageQueryContext; import org.apache.druid.query.QueryContext; import org.apache.druid.server.DruidNode; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java index fe68b2737ef3..23d404354701 100644 --- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java @@ -58,7 +58,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index efaf09232c35..d02f04b46d7e 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -71,11 +71,7 @@ public class BrokerServerView implements TimelineServerView { private static final Logger log = new Logger(BrokerServerView.class); - private final Object lock = new Object(); - private final ConcurrentMap clients = new ConcurrentHashMap<>(); - private final Map selectors = new HashMap<>(); - private final Map> timelines = new HashMap<>(); private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); private final QueryToolChestWarehouse warehouse; @@ -87,9 +83,14 @@ public class BrokerServerView implements TimelineServerView private final ServiceEmitter emitter; private final BrokerSegmentWatcherConfig segmentWatcherConfig; private final Predicate> segmentFilter; - private final CountDownLatch initialized = new CountDownLatch(1); + protected final Object lock = new Object(); + + protected final Map selectors = new HashMap<>(); + + protected final Map> timelines = new HashMap<>(); + @Inject public BrokerServerView( final QueryToolChestWarehouse warehouse, @@ -431,24 +432,4 @@ public List getDruidServers() .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) .collect(Collectors.toList()); } - - public Map getSegmentMetadata() - { - return ImmutableMap.copyOf(selectors); - } - - Object getLock() - { - return lock; - } - - Map> getTimelines() - { - return timelines; - } - - Map getSelectors() - { - return selectors; - } } diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index ade760710def..b5f8973519f9 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -75,9 +75,9 @@ public boolean isAwaitInitializationOnStart() public VersionedIntervalTimeline getTimeline(DataSource dataSource) { String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (getLock()) { + synchronized (lock) { // build a new timeline? - VersionedIntervalTimeline timeline = getTimelines().get(table); + VersionedIntervalTimeline timeline = timelines.get(table); Collection x = timeline.iterateAllObjects(); VersionedIntervalTimeline newTimeline = new VersionedIntervalTimeline<>(Comparator.naturalOrder()); newTimeline.addAll(x.stream().map(v -> new VersionedIntervalTimeline.PartitionChunkEntry( @@ -90,7 +90,7 @@ public VersionedIntervalTimeline getTimeline(DataSource @Override public Map getSegmentLoadInfos() { - return CollectionUtils.mapValues(getSelectors(), ServerSelector::toSegmentLoadInfo); + return CollectionUtils.mapValues(selectors, ServerSelector::toSegmentLoadInfo); } @Override diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 3c20b4bcd779..56328fbc3dec 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -176,28 +176,10 @@ public class SegmentMetadataCache // For awaitInitialization. private final CountDownLatch initialized = new CountDownLatch(1); - /** - * This lock coordinates the access from multiple threads to those variables guarded by this lock. - * Currently, there are 2 threads that can access these variables. - * - * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceSchema} if necessary - * based on the information collected via timeline callbacks. - */ - private final Object lock = new Object(); - // All mutable segments. @GuardedBy("lock") private final TreeSet mutableSegments = new TreeSet<>(SEGMENT_ORDER); - // All dataSources that need tables regenerated. - @GuardedBy("lock") - private final Set dataSourcesNeedingRebuild = new HashSet<>(); - - // All segments that need to be refreshed. - @GuardedBy("lock") - private final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); - // Configured context to attach to internally generated queries. private final InternalQueryConfig internalQueryConfig; @@ -214,6 +196,24 @@ public class SegmentMetadataCache */ private int totalSegments = 0; + /** + * This lock coordinates the access from multiple threads to those variables guarded by this lock. + * Currently, there are 2 threads that can access these variables. + * + * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceSchema} if necessary + * based on the information collected via timeline callbacks. + */ + protected final Object lock = new Object(); + + // All dataSources that need tables regenerated. + @GuardedBy("lock") + protected final Set dataSourcesNeedingRebuild = new HashSet<>(); + + // All segments that need to be refreshed. + @GuardedBy("lock") + protected final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); + @Inject public SegmentMetadataCache( final QueryLifecycleFactory queryLifecycleFactory, @@ -281,17 +281,16 @@ public ServerView.CallbackAction serverSegmentRemoved( ); } - public void removeFromTable(String s) + protected void removeFromTable(String s) { tables.remove(s); } - public boolean tablesContains(String s) + protected boolean tablesContains(String s) { return tables.containsKey(s); } - private void startCacheExec() { cacheExec.submit( @@ -901,7 +900,7 @@ public Set getDataSourcesNeedingRebuild() } } - public Object getLock() + Object getLock() { return lock; } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 80aba42ecddc..b1932485b1ad 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -9,7 +9,9 @@ public class SegmentMetadataCacheConfig private boolean awaitInitializationOnStart = false; @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); - private boolean enabled = false; + @JsonProperty + private final boolean enabled = false; + @JsonProperty private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); diff --git a/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java b/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java index bcaab5c255fb..b2cde828092e 100644 --- a/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java +++ b/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java @@ -22,10 +22,12 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; import java.util.List; +import java.util.Set; public class NoopCoordinatorClient implements CoordinatorClient { @@ -47,6 +49,12 @@ public ListenableFuture> fetchUsedSegments(String dataSource, throw new UnsupportedOperationException(); } + @Override + public ListenableFuture> fetchDataSourceSchema(Set datasources) + { + throw new UnsupportedOperationException(); + } + @Override public CoordinatorClient withRetryPolicy(ServiceRetryPolicy retryPolicy) { diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index dbc104eafa2b..92ad857476e1 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -21,8 +21,6 @@ import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; @@ -42,7 +40,6 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.metadata.PhysicalDatasourceMetadata; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.TableDataSource; @@ -51,15 +48,12 @@ import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.sql.calcite.table.DatasourceTable; -import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.DataSegment.PruneSpecsHolder; import org.apache.druid.timeline.SegmentId; @@ -135,10 +129,8 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV throws InterruptedException, ExecutionException, TimeoutException { schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -146,7 +138,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV ) { @Override - DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(final String dataSource) + public DataSourceSchema buildDruidTable(final String dataSource) { doInLock(() -> { try { @@ -245,10 +237,8 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() throws InterruptedException, ExecutionException, TimeoutException { schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -256,7 +246,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() ) { @Override - DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(final String dataSource) + public DataSourceSchema buildDruidTable(final String dataSource) { doInLock(() -> { try { diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index acf2c6b76c9d..a391ba9fb74c 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -19,21 +19,28 @@ package org.apache.druid.segment.metadata; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.MapInputRowParser; +import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.io.Closer; -import org.apache.druid.query.DataSource; -import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.query.DefaultGenericQueryMetricsFactory; +import org.apache.druid.query.DefaultQueryConfig; +import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerFactoryConglomerate; -import org.apache.druid.segment.join.JoinConditionAnalysis; -import org.apache.druid.segment.join.Joinable; -import org.apache.druid.segment.join.JoinableFactory; -import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; -import org.apache.druid.server.SegmentManager; -import org.easymock.EasyMock; +import org.apache.druid.server.log.TestRequestLogger; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.AuthConfig; +import org.apache.druid.server.security.AuthTestUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -42,36 +49,59 @@ import java.io.IOException; import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; +import java.util.Map; -public abstract class SegmentMetadataCacheCommon extends CalciteTestBase +public abstract class SegmentMetadataCacheCommon { + public static final String DATASOURCE1 = "foo"; + public static final String DATASOURCE2 = "foo2"; + public static final String DATASOURCE3 = "numfoo"; + public static final String DATASOURCE4 = "foo4"; + public static final String DATASOURCE5 = "lotsocolumns"; + public static final String BROADCAST_DATASOURCE = "broadcast"; + public static final String FORBIDDEN_DATASOURCE = "forbiddenDatasource"; + public static final String SOME_DATASOURCE = "some_datasource"; + public static final String TIMESTAMP_COLUMN = "t"; + private static final InputRowSchema FOO_SCHEMA = new InputRowSchema( + new TimestampSpec(TIMESTAMP_COLUMN, "iso", null), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim2", "dim3")) + ), + null + ); + static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - static final List ROWS1 = ImmutableList.of( - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) + final List ROWS1 = ImmutableList.of( + createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), + createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), + createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) ); - static final List ROWS2 = ImmutableList.of( - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) + final List ROWS2 = ImmutableList.of( + createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), + createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), + createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) ); static QueryRunnerFactoryConglomerate conglomerate; static Closer resourceCloser; - CountDownLatch getDatasourcesLatch = new CountDownLatch(1); + static QueryToolChestWarehouse queryToolChestWarehouse; @BeforeClass public static void setUpClass() { resourceCloser = Closer.create(); conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); + queryToolChestWarehouse = new QueryToolChestWarehouse() + { + @Override + public > QueryToolChest getToolChest(final QueryType query) + { + return conglomerate.findFactory(query).getToolchest(); + } + }; } @AfterClass @@ -83,44 +113,32 @@ public static void tearDownClass() throws IOException @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - SegmentManager segmentManager; - Set segmentDataSourceNames; - Set joinableDataSourceNames; - JoinableFactory globalTableJoinable; @Before public void setUpCommon() { - segmentDataSourceNames = Sets.newConcurrentHashSet(); - joinableDataSourceNames = Sets.newConcurrentHashSet(); + } - segmentManager = new SegmentManager(EasyMock.createMock(SegmentLoader.class)) - { - @Override - public Set getDataSourceNames() - { - getDatasourcesLatch.countDown(); - return segmentDataSourceNames; - } - }; + InputRow createRow(final ImmutableMap map) + { + return MapInputRowParser.parse(FOO_SCHEMA, (Map) map); + } - globalTableJoinable = new JoinableFactory() - { - @Override - public boolean isDirectlyJoinable(DataSource dataSource) - { - return dataSource instanceof GlobalTableDataSource && - joinableDataSourceNames.contains(((GlobalTableDataSource) dataSource).getName()); - } + InputRow createRow(final ImmutableMap map, InputRowSchema inputRowSchema) + { + return MapInputRowParser.parse(inputRowSchema, (Map) map); + } - @Override - public Optional build( - DataSource dataSource, - JoinConditionAnalysis condition - ) - { - return Optional.empty(); - } - }; + QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) { + return new QueryLifecycleFactory( + queryToolChestWarehouse, + walker, + new DefaultGenericQueryMetricsFactory(), + new NoopServiceEmitter(), + new TestRequestLogger(), + new AuthConfig(), + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of())) + ); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java similarity index 75% rename from sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java index 29de2639ae4a..16567719dffc 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.planner; +package org.apache.druid.segment.metadata; import com.google.common.collect.ImmutableList; import com.google.inject.Injector; @@ -37,21 +37,22 @@ */ public class SegmentMetadataCacheConfigTest { + private static final String CONFIG_BASE = "druid.coordinator.segmentMetadataCache"; + @Test public void testDefaultConfig() { final Injector injector = createInjector(); final JsonConfigProvider provider = JsonConfigProvider.of( - CalcitePlannerModule.CONFIG_BASE, + CONFIG_BASE, SegmentMetadataCacheConfig.class ); + final Properties properties = new Properties(); provider.inject(properties, injector.getInstance(JsonConfigurator.class)); final SegmentMetadataCacheConfig config = provider.get(); Assert.assertTrue(config.isAwaitInitializationOnStart()); - Assert.assertFalse(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); - Assert.assertEquals(60_000, config.getMetadataSegmentPollPeriod()); Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } @@ -60,24 +61,20 @@ public void testCustomizedConfig() { final Injector injector = createInjector(); final JsonConfigProvider provider = JsonConfigProvider.of( - CalcitePlannerModule.CONFIG_BASE, + CONFIG_BASE, SegmentMetadataCacheConfig.class ); final Properties properties = new Properties(); properties.setProperty( - CalcitePlannerModule.CONFIG_BASE + ".metadataColumnTypeMergePolicy", + CONFIG_BASE + ".metadataColumnTypeMergePolicy", "latestInterval" ); - properties.setProperty(CalcitePlannerModule.CONFIG_BASE + ".metadataRefreshPeriod", "PT2M"); - properties.setProperty(CalcitePlannerModule.CONFIG_BASE + ".metadataSegmentPollPeriod", "15000"); - properties.setProperty(CalcitePlannerModule.CONFIG_BASE + ".metadataSegmentCacheEnable", "true"); - properties.setProperty(CalcitePlannerModule.CONFIG_BASE + ".awaitInitializationOnStart", "false"); + properties.setProperty(CONFIG_BASE + ".metadataRefreshPeriod", "PT2M"); + properties.setProperty(CONFIG_BASE + ".awaitInitializationOnStart", "false"); provider.inject(properties, injector.getInstance(JsonConfigurator.class)); final SegmentMetadataCacheConfig config = provider.get(); Assert.assertFalse(config.isAwaitInitializationOnStart()); - Assert.assertTrue(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); - Assert.assertEquals(15_000, config.getMetadataSegmentPollPeriod()); Assert.assertEquals( new SegmentMetadataCache.FirstTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy() @@ -89,7 +86,7 @@ private Injector createInjector() return GuiceInjectors.makeStartupInjectorWithModules( ImmutableList.of( binder -> { - JsonConfigProvider.bind(binder, CalcitePlannerModule.CONFIG_BASE, SegmentMetadataCacheConfig.class); + JsonConfigProvider.bind(binder, CONFIG_BASE, SegmentMetadataCacheConfig.class); } ) ); diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 5f1fccbd15f3..dc18cfb79741 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -26,8 +26,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -37,7 +37,6 @@ import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.aggregation.CountAggregatorFactory; @@ -55,11 +54,11 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryResponse; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -95,7 +94,7 @@ public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon private static final int WAIT_TIMEOUT_SECS = 6; private SpecificSegmentsQuerySegmentWalker walker; - private TestServerInventoryView serverView; + private TestTimelineServerView serverView; private List druidServers; private SegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); @@ -120,7 +119,7 @@ public void setUp() throws Exception .withRollup(false) .build() ) - .rows(SegmentMetadataCacheCommon.ROWS1) + .rows(ROWS1) .buildMMappedIndex(); final QueryableIndex index2 = IndexBuilder.create() @@ -132,7 +131,7 @@ public void setUp() throws Exception .withRollup(false) .build() ) - .rows(SegmentMetadataCacheCommon.ROWS2) + .rows(ROWS2) .buildMMappedIndex(); final InputRowSchema rowSchema = new InputRowSchema( @@ -141,7 +140,7 @@ public void setUp() throws Exception null ); final List autoRows1 = ImmutableList.of( - TestDataBuilder.createRow( + createRow( ImmutableMap.builder() .put("t", "2023-01-01T00:00Z") .put("numbery", 1.1f) @@ -154,7 +153,7 @@ public void setUp() throws Exception ) ); final List autoRows2 = ImmutableList.of( - TestDataBuilder.createRow( + createRow( ImmutableMap.builder() .put("t", "2023-01-02T00:00Z") .put("numbery", 1L) @@ -166,6 +165,7 @@ public void setUp() throws Exception rowSchema ) ); + final QueryableIndex indexAuto1 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) @@ -208,7 +208,7 @@ public void setUp() throws Exception walker = new SpecificSegmentsQuerySegmentWalker(SegmentMetadataCacheCommon.conglomerate).add( DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE1) + .dataSource(DATASOURCE1) .interval(Intervals.of("2000/P1Y")) .version("1") .shardSpec(new LinearShardSpec(0)) @@ -217,7 +217,7 @@ public void setUp() throws Exception index1 ).add( DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE1) + .dataSource(DATASOURCE1) .interval(Intervals.of("2001/P1Y")) .version("1") .shardSpec(new LinearShardSpec(0)) @@ -226,7 +226,7 @@ public void setUp() throws Exception index2 ).add( DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE2) + .dataSource(DATASOURCE2) .interval(index2.getDataInterval()) .version("1") .shardSpec(new LinearShardSpec(0)) @@ -235,7 +235,7 @@ public void setUp() throws Exception index2 ).add( DataSegment.builder() - .dataSource(CalciteTests.SOME_DATASOURCE) + .dataSource(SOME_DATASOURCE) .interval(Intervals.of("2023-01-01T00Z/P1D")) .version("1") .shardSpec(new LinearShardSpec(1)) @@ -244,7 +244,7 @@ public void setUp() throws Exception indexAuto1 ).add( DataSegment.builder() - .dataSource(CalciteTests.SOME_DATASOURCE) + .dataSource(SOME_DATASOURCE) .interval(Intervals.of("2023-01-02T00Z/P1D")) .version("1") .shardSpec(new LinearShardSpec(1)) @@ -266,7 +266,7 @@ public void setUp() throws Exception PruneSpecsHolder.DEFAULT ); final List realtimeSegments = ImmutableList.of(segment1); - serverView = new TestServerInventoryView(walker.getSegments(), realtimeSegments); + serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); druidServers = serverView.getDruidServers(); } @@ -279,13 +279,8 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheCon { Preconditions.checkState(runningSchema == null); runningSchema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory( - ImmutableSet.of(globalTableJoinable), - ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) - ), config, new NoopEscalator(), new InternalQueryConfig(), @@ -293,9 +288,9 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheCon ) { @Override - protected DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(String dataSource) + public DataSourceSchema buildDruidTable(String dataSource) { - DatasourceTable.PhysicalDatasourceMetadata table = super.buildDruidTable(dataSource); + DataSourceSchema table = super.buildDruidTable(dataSource); buildTableLatch.countDown(); return table; } @@ -317,13 +312,8 @@ public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedE { Preconditions.checkState(runningSchema == null); runningSchema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory( - ImmutableSet.of(globalTableJoinable), - ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) - ), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -364,68 +354,31 @@ public void tearDown() throws Exception public void testGetTableMap() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); + Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); final Set tableNames = schema.getDatasourceNames(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), tableNames); + Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), tableNames); } @Test public void testSchemaInit() throws InterruptedException { SegmentMetadataCache schema2 = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema2.getDatasourceNames()); + Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema2.getDatasourceNames()); } @Test public void testGetTableMapFoo() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource("foo"); - final DruidTable fooTable = new DatasourceTable(fooDs); - final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); - final List fields = rowType.getFieldList(); - - Assert.assertEquals(6, fields.size()); - - Assert.assertEquals("__time", fields.get(0).getName()); - Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); - - Assert.assertEquals("dim2", fields.get(1).getName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName()); - - Assert.assertEquals("m1", fields.get(2).getName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getSqlTypeName()); - - Assert.assertEquals("dim1", fields.get(3).getName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName()); - - Assert.assertEquals("cnt", fields.get(4).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getSqlTypeName()); - - Assert.assertEquals("unique_dim1", fields.get(5).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); + final DataSourceSchema fooDs = schema.getDatasource("foo"); } @Test public void testGetTableMapFoo2() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource("foo2"); - final DruidTable fooTable = new DatasourceTable(fooDs); - final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); - final List fields = rowType.getFieldList(); - - Assert.assertEquals(3, fields.size()); - - Assert.assertEquals("__time", fields.get(0).getName()); - Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); - - Assert.assertEquals("dim2", fields.get(1).getName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName()); - - Assert.assertEquals("m1", fields.get(2).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(2).getType().getSqlTypeName()); + final DataSourceSchema fooDs = schema.getDatasource("foo2"); } @Test @@ -442,41 +395,8 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli } } ); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource(CalciteTests.SOME_DATASOURCE); - final DruidTable table = new DatasourceTable(fooDs); - final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); - final List fields = rowType.getFieldList(); - - Assert.assertEquals(9, fields.size()); - - Assert.assertEquals("__time", fields.get(0).getName()); - Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + final DataSourceSchema fooDs = schema.getDatasource(SOME_DATASOURCE); - Assert.assertEquals("numbery", fields.get(1).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(1).getType().getSqlTypeName()); - - Assert.assertEquals("numberyArrays", fields.get(2).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(2).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("stringy", fields.get(3).getName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName()); - - Assert.assertEquals("array", fields.get(4).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(4).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("nested", fields.get(5).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); - - Assert.assertEquals("cnt", fields.get(6).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(6).getType().getSqlTypeName()); - - Assert.assertEquals("m1", fields.get(7).getName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(7).getType().getSqlTypeName()); - - Assert.assertEquals("unique_dim1", fields.get(8).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(8).getType().getSqlTypeName()); } @Test @@ -485,42 +405,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource(CalciteTests.SOME_DATASOURCE); - final DruidTable table = new DatasourceTable(fooDs); - final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); - final List fields = rowType.getFieldList(); - - Assert.assertEquals(9, fields.size()); - - Assert.assertEquals("__time", fields.get(0).getName()); - Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); - - Assert.assertEquals("numbery", fields.get(1).getName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(1).getType().getSqlTypeName()); - - Assert.assertEquals("numberyArrays", fields.get(2).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(2).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("stringy", fields.get(3).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(3).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("array", fields.get(4).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(4).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(4).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("nested", fields.get(5).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); - - Assert.assertEquals("cnt", fields.get(6).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(6).getType().getSqlTypeName()); - - Assert.assertEquals("m1", fields.get(7).getName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(7).getType().getSqlTypeName()); - - Assert.assertEquals("unique_dim1", fields.get(8).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(8).getType().getSqlTypeName()); + final DataSourceSchema fooDs = schema.getDatasource(SOME_DATASOURCE); } @@ -691,10 +576,8 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -734,10 +617,8 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -781,10 +662,8 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -825,10 +704,8 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -866,10 +743,8 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -924,10 +799,8 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -985,10 +858,8 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -1020,10 +891,8 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -1068,10 +937,8 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -1135,12 +1002,8 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte { SegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema3.getDatasource("foo"); + DataSourceSchema fooTable = schema3.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertFalse(fooTable.isJoinable()); - Assert.assertFalse(fooTable.isBroadcast()); markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); @@ -1157,8 +1020,6 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte 100L, PruneSpecsHolder.DEFAULT ); - segmentDataSourceNames.add("foo"); - joinableDataSourceNames.add("foo"); serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); // wait for build twice @@ -1169,16 +1030,10 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte fooTable = schema3.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertTrue(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertTrue(fooTable.isJoinable()); - Assert.assertTrue(fooTable.isBroadcast()); // now remove it markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); - joinableDataSourceNames.remove("foo"); - segmentDataSourceNames.remove("foo"); serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1190,10 +1045,6 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte fooTable = schema3.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertFalse(fooTable.isJoinable()); - Assert.assertFalse(fooTable.isBroadcast()); } @Test @@ -1201,12 +1052,8 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw { SegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getDatasource("foo"); + DataSourceSchema fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertFalse(fooTable.isJoinable()); - Assert.assertFalse(fooTable.isBroadcast()); markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); @@ -1223,7 +1070,6 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw 100L, PruneSpecsHolder.DEFAULT ); - segmentDataSourceNames.add("foo"); serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1235,17 +1081,10 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - // Should not be a GlobalTableDataSource for now, because isGlobal is couple with joinability. Ideally this will be - // changed in the future and we should expect. - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertTrue(fooTable.isBroadcast()); - Assert.assertFalse(fooTable.isJoinable()); // now remove it markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); - segmentDataSourceNames.remove("foo"); serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1257,10 +1096,6 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertFalse(fooTable.isBroadcast()); - Assert.assertFalse(fooTable.isJoinable()); } /** @@ -1309,11 +1144,6 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception SegmentMetadataCache mySchema = new SegmentMetadataCache( factoryMock, serverView, - segmentManager, - new MapJoinableFactory( - ImmutableSet.of(globalTableJoinable), - ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) - ), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), internalQueryConfig, @@ -1446,10 +1276,8 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestServerInventoryView.java b/server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java similarity index 96% rename from sql/src/test/java/org/apache/druid/sql/calcite/util/TestServerInventoryView.java rename to server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java index d74b36739cfd..42c219665909 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestServerInventoryView.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.util; +package org.apache.druid.segment.metadata; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -44,7 +44,7 @@ /** * This class is used for testing and benchmark */ -public class TestServerInventoryView implements TimelineServerView +public class TestTimelineServerView implements TimelineServerView { private static final DruidServerMetadata DUMMY_SERVER = new DruidServerMetadata( "dummy", @@ -80,12 +80,12 @@ public class TestServerInventoryView implements TimelineServerView private List> segmentCallbackExecs = new ArrayList<>(); private List> timelineCallbackExecs = new ArrayList<>(); - public TestServerInventoryView(List segments) + public TestTimelineServerView(List segments) { this.segments.addAll(segments); } - public TestServerInventoryView(List segments, List realtimeSegments) + public TestTimelineServerView(List segments, List realtimeSegments) { this.segments.addAll(segments); this.realtimeSegments.addAll(realtimeSegments); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java b/server/src/test/java/org/apache/druid/server/SpecificSegmentsQuerySegmentWalker.java similarity index 97% rename from sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java rename to server/src/test/java/org/apache/druid/server/SpecificSegmentsQuerySegmentWalker.java index b43a65159567..c111cc9f7f4d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java +++ b/server/src/test/java/org/apache/druid/server/SpecificSegmentsQuerySegmentWalker.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.util; +package org.apache.druid.server; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; @@ -44,9 +44,6 @@ import org.apache.druid.segment.SegmentWrangler; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.join.JoinableFactoryWrapper; -import org.apache.druid.server.ClientQuerySegmentWalker; -import org.apache.druid.server.QueryScheduler; -import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.VersionedIntervalTimeline; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index b55dc855572a..9fa0f3938b81 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -61,7 +61,7 @@ public class BrokerSegmentMetadataCache extends SegmentMetadataCache public BrokerSegmentMetadataCache( final QueryLifecycleFactory queryLifecycleFactory, final TimelineServerView serverView, - final SegmentMetadataCacheConfig config, + final BrokerSegmentMetadataCacheConfig config, final Escalator escalator, final InternalQueryConfig internalQueryConfig, final ServiceEmitter emitter, @@ -118,17 +118,17 @@ public void refresh(final Set segmentsToRefresh, final Set da // Refresh the segments. final Set refreshed = refreshSegments(segmentsToRefresh); - synchronized (getLock()) { + synchronized (lock) { // Add missing segments back to the refresh list. getSegmentsNeedingRefresh().addAll(Sets.difference(segmentsToRefresh, refreshed)); // Compute the list of dataSources to rebuild tables for. - dataSourcesToRebuild.addAll(getDataSourcesNeedingRebuild()); + dataSourcesToRebuild.addAll(dataSourcesNeedingRebuild); refreshed.forEach(segment -> dataSourcesToRebuild.add(segment.getDataSource())); // Remove those dataSource for which we received schema from the Coordinator. dataSourcesToRebuild.removeAll(polledDataSourceSchema.keySet()); - getDataSourcesNeedingRebuild().clear(); + dataSourcesNeedingRebuild.clear(); } // Rebuild the dataSources. @@ -163,13 +163,13 @@ public Set getDatasourceNames() } @Override - public void removeFromTable(String s) + protected void removeFromTable(String s) { tables.remove(s); } @Override - public boolean tablesContains(String s) + protected boolean tablesContains(String s) { return tables.containsKey(s); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 834b2d7079e1..2554753f79c3 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -33,6 +33,11 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig @JsonProperty private boolean awaitInitializationOnStart = true; + public static BrokerSegmentMetadataCacheConfig create() + { + return new BrokerSegmentMetadataCacheConfig(); + } + public boolean isMetadataSegmentCacheEnable() { return metadataSegmentCacheEnable; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index fc20c38d00a4..9ef381b10b3f 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -90,8 +90,6 @@ public class BrokerSegmentMetadataView @MonotonicNonNull private volatile ImmutableSortedSet segmentMetadata = null; - private final BrokerServerView brokerServerView; - private final BrokerSegmentMetadataCache segmentMetadataCache; /** @@ -111,7 +109,6 @@ public BrokerSegmentMetadataView( final ObjectMapper objectMapper, final BrokerSegmentWatcherConfig segmentWatcherConfig, final BrokerSegmentMetadataCacheConfig config, - final BrokerServerView brokerServerView, final BrokerSegmentMetadataCache segmentMetadataCache ) { @@ -127,7 +124,6 @@ public BrokerSegmentMetadataView( this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() .expireAfterAccess(10, TimeUnit.MINUTES) .build(); - this.brokerServerView = brokerServerView; this.segmentMetadataCache = segmentMetadataCache; } @@ -167,7 +163,6 @@ protected Iterator getSegmentTableView() { final ImmutableSortedSet segments = getSegmentMetadata(); final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - final Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); final List segmentsTableViews = new ArrayList<>(); Set seenSegments = new HashSet<>(); @@ -183,10 +178,6 @@ protected Iterator getSegmentTableView() numReplicas = availableSegmentMetadata.getNumReplicas(); numRows = availableSegmentMetadata.getNumRows(); isAvailable = 1L; - } else if (brokerSegmentMetadata.containsKey(segmentId)) { - ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); - numReplicas = serverSelector.getAllServers().size(); - isAvailable = 1L; } // Prefer numRows & realtime status info returned from Coordinator. @@ -244,7 +235,7 @@ private void pollSegmentMetadata() segmentMetadataCachePopulated.countDown(); } - ImmutableSortedSet getSegmentMetadata() + private ImmutableSortedSet getSegmentMetadata() { if (isMetadataSegmentCacheEnabled) { Uninterruptibles.awaitUninterruptibly(segmentMetadataCachePopulated); diff --git a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java index 9aab1faeaaf4..cad98cd8409e 100644 --- a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java @@ -58,7 +58,7 @@ import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.http.SqlQuery; import org.easymock.EasyMock; import org.hamcrest.MatcherAssert; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 1461c1e5ad8f..bcc2dddb4c9e 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -88,7 +88,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.guice.SqlModule; import org.eclipse.jetty.server.Server; import org.joda.time.DateTime; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java index a1083ab30a7b..1df137e772c3 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java @@ -47,7 +47,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 3c73319898e4..ecd104d0b669 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -97,7 +97,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder; import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index 406f94b46435..d366845bb9b2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -69,7 +69,7 @@ import org.apache.druid.segment.virtual.NestedFieldVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.sql.calcite.filtration.Filtration; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java index 762ae621d1fd..27a099b1c479 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java @@ -44,7 +44,7 @@ import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMediumFactory; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.junit.Assert; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java index 2ac0574c2789..a439f1d19ea4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java @@ -48,7 +48,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index c1a0d5047ea7..974edbd3a588 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.common.io.Closer; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; @@ -34,8 +33,8 @@ import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; -import org.apache.druid.sql.calcite.util.TestServerInventoryView; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.segment.metadata.TestTimelineServerView; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; @@ -44,25 +43,27 @@ public class DruidSchemaNoDataInitTest extends CalciteTestBase { - private static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create(); + private static final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create(); @Test public void testInitializationWithNoData() throws Exception { try (final Closer closer = Closer.create()) { final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(closer); - final SegmentMetadataCache cache = new SegmentMetadataCache( + final BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( CalciteTests.createMockQueryLifecycleFactory( new SpecificSegmentsQuerySegmentWalker(conglomerate), conglomerate ), - new TestServerInventoryView(Collections.emptyList()), - new SegmentManager(EasyMock.createMock(SegmentLoader.class)), - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), + new TestTimelineServerView(Collections.emptyList()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), - new NoopServiceEmitter() + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder( + new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), + new SegmentManager(EasyMock.createMock(SegmentLoader.class))), + null ); cache.start(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 659be26029e4..b17eb7713688 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -65,7 +65,6 @@ import org.apache.druid.java.util.http.client.response.HttpResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -97,16 +96,16 @@ import org.apache.druid.server.security.ResourceType; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable; +import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; -import org.apache.druid.sql.calcite.util.TestServerInventoryView; +import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.SegmentStatusInCluster; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpResponse; @@ -134,7 +133,7 @@ public class SystemSchemaTest extends CalciteTestBase { - private static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create(); + private static final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create(); private static final List ROWS1 = ImmutableList.of( TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), @@ -255,14 +254,15 @@ public void setUp() throws Exception BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - new TestServerInventoryView(walker.getSegments(), realtimeSegments), + new TestTimelineServerView(walker.getSegments(), realtimeSegments), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), new PhysicalDatasourceMetadataBuilder( new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - new SegmentManager(EasyMock.createMock(SegmentLoader.class))), + new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + ), new NoopCoordinatorClient() ); cache.start(); @@ -569,15 +569,16 @@ public void testGetTableMap() public void testSegmentsTable() throws Exception { final SegmentsTable segmentsTable = new SegmentsTable(metadataView, new ObjectMapper(), authMapper); - final Set publishedSegments = new HashSet<>(Arrays.asList( - new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 2L, true), - new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 2L, true), - new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 2L, true), - new SegmentStatusInCluster(segment1, true, 2, 2L, true), - new SegmentStatusInCluster(segment2, false, 0, 2L, true) + + final Set segmentTableViews = new HashSet<>(Arrays.asList( + new SegmentTableView(publishedCompactedSegment1, 1L, 0L, 1, 2, 2, true), + new SegmentTableView(publishedCompactedSegment2, 1L, 0L, 0, 2, 0, false), + new SegmentTableView(publishedUncompactedSegment3, 1L, 0L, 1, 2, 2, false), + new SegmentTableView(segment1, 1L, 0L, 1, 2, 2, true), + new SegmentTableView(segment2, 1L, 0L, 1, 2, 0, false) )); - EasyMock.expect(metadataView.getSegmentMetadata()).andReturn(publishedSegments.iterator()).once(); + EasyMock.expect(metadataView.getSegmentTableView()).andReturn(segmentTableViews.iterator()).once(); EasyMock.replay(client, request, responseHolder, responseHandler, metadataView); DataContext dataContext = createDataContext(Users.SUPER); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index f182bd5dab58..3f5e9b14f5e9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -30,8 +31,11 @@ import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ServerInventoryView; import org.apache.druid.client.ServerView; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.client.indexing.NoopOverlordClient; import org.apache.druid.discovery.DiscoveryDruidNode; import org.apache.druid.discovery.DruidLeaderClient; @@ -49,10 +53,16 @@ import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.join.JoinableFactoryWrapper; +import org.apache.druid.segment.join.MapJoinableFactory; +import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.TestTimelineServerView; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.DruidNode; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; @@ -68,14 +78,17 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; +import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataBuilder; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.timeline.DataSegment; +import org.easymock.EasyMock; import org.joda.time.Duration; import javax.annotation.Nullable; @@ -330,7 +343,7 @@ public static DruidOperatorTable createOperatorTable() } public static SystemSchema createMockSystemSchema( - final DruidSchema druidSchema, + final QueryLifecycleFactory queryLifecycleFactory, final SpecificSegmentsQuerySegmentWalker walker, final AuthorizerMapper authorizerMapper ) @@ -370,15 +383,29 @@ public ListenableFuture findCurrentLeader() } }; + TimelineServerView timelineServerView = new TestTimelineServerView(walker.getSegments()); + return new SystemSchema( - druidSchema, new BrokerSegmentMetadataView( druidLeaderClient, getJsonMapper(), new BrokerSegmentWatcherConfig(), - SegmentMetadataCacheConfig.create() + BrokerSegmentMetadataCacheConfig.create(), + new BrokerSegmentMetadataCache( + queryLifecycleFactory, + timelineServerView, + BrokerSegmentMetadataCacheConfig.create(), + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder( + new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), + new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + ), + new NoopCoordinatorClient() + ) ), - new TestServerInventoryView(walker.getSegments()), + timelineServerView, new FakeServerInventoryView(), authorizerMapper, druidLeaderClient, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 1e2b9aab25bf..9f540d634d06 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -42,6 +42,8 @@ import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.TestTimelineServerView; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; @@ -57,6 +59,8 @@ import org.apache.druid.sql.calcite.planner.PlannerFactory; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.schema.DruidSchemaManager; @@ -68,7 +72,7 @@ import org.apache.druid.sql.calcite.schema.NamedSystemSchema; import org.apache.druid.sql.calcite.schema.NamedViewSchema; import org.apache.druid.sql.calcite.schema.NoopDruidSchemaManager; -import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataBuilder; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.sql.calcite.schema.ViewSchema; import org.apache.druid.sql.calcite.view.ViewManager; @@ -143,11 +147,10 @@ public static DruidSchemaCatalog createMockRootSchema( injector, conglomerate, walker, - plannerConfig, druidSchemaManager ); SystemSchema systemSchema = - CalciteTests.createMockSystemSchema(druidSchema, walker, authorizerMapper); + CalciteTests.createMockSystemSchema(createMockQueryLifecycleFactory(walker, conglomerate), walker, authorizerMapper); LookupSchema lookupSchema = createMockLookupSchema(injector); ViewSchema viewSchema = viewManager != null ? new ViewSchema(viewManager) : null; @@ -186,7 +189,7 @@ public static DruidSchemaCatalog createMockRootSchema( public static DruidSchemaCatalog createMockRootSchema( final Injector injector, - final QueryRunnerFactoryConglomerate conglomerate, + final QueryRunnerFactoryConglomerate conglomerate, final SpecificSegmentsQuerySegmentWalker walker, final PlannerConfig plannerConfig, final AuthorizerMapper authorizerMapper @@ -207,26 +210,27 @@ private static DruidSchema createMockSchema( final Injector injector, final QueryRunnerFactoryConglomerate conglomerate, final SpecificSegmentsQuerySegmentWalker walker, - final PlannerConfig plannerConfig, final DruidSchemaManager druidSchemaManager ) { - final SegmentMetadataCache cache = new SegmentMetadataCache( + final BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( createMockQueryLifecycleFactory(walker, conglomerate), - new TestServerInventoryView(walker.getSegments()), - new SegmentManager(EasyMock.createMock(SegmentLoader.class)) - { - @Override - public Set getDataSourceNames() - { - return ImmutableSet.of(CalciteTests.BROADCAST_DATASOURCE); - } - }, - createDefaultJoinableFactory(injector), - SegmentMetadataCacheConfig.create(), + new TestTimelineServerView(walker.getSegments()), + BrokerSegmentMetadataCacheConfig.create(), CalciteTests.TEST_AUTHENTICATOR_ESCALATOR, new InternalQueryConfig(), - new NoopServiceEmitter() + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder( + createDefaultJoinableFactory(injector), + new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + { + @Override + public Set getDataSourceNames() + { + return ImmutableSet.of(CalciteTests.BROADCAST_DATASOURCE); + } + }), + null ); try { @@ -274,5 +278,5 @@ public static DruidOperatorTable createOperatorTable(final Injector injector) public static LookupSchema createMockLookupSchema(final Injector injector) { return new LookupSchema(injector.getInstance(LookupExtractorFactoryContainerProvider.class)); - } **/ + } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 3467d71bb627..7e012b435811 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -40,6 +40,7 @@ import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.segment.join.JoinableFactoryWrapper; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java index b2f93340dbf7..edfa2fc2d334 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java @@ -61,6 +61,7 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.join.table.IndexedTableJoinable; import org.apache.druid.segment.join.table.RowBasedIndexedTable; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 98f1f7b1cab5..eefbe389e62d 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -103,7 +103,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.After; From 95389b177f5dd3da2c3686d17213e118ce82a489 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 8 Sep 2023 21:49:21 +0530 Subject: [PATCH 10/82] Fix unit tests and add test for getAllUsedSegments --- .../timeline/SegmentStatusInCluster.java | 22 +-- .../druid/client/CoordinatorServerView.java | 19 ++ .../druid/client/CoordinatorTimeline.java | 21 +- .../client/coordinator/CoordinatorClient.java | 4 +- .../coordinator/CoordinatorClientImpl.java | 8 +- ...Schema.java => DataSourceInformation.java} | 6 +- .../metadata/SegmentMetadataCache.java | 16 +- .../druid/server/http/MetadataResource.java | 48 +++-- .../coordinator/NoopCoordinatorClient.java | 4 +- .../SegmentDataCacheConcurrencyTest.java | 4 +- .../SegmentMetadataCacheConfigTest.java | 2 +- .../metadata/SegmentMetadataCacheTest.java | 16 +- .../server/http/MetadataResourceTest.java | 133 +++++++++++-- .../schema/BrokerSegmentMetadataCache.java | 15 +- .../schema/BrokerSegmentMetadataView.java | 45 +++-- .../PhysicalDatasourceMetadataBuilder.java | 5 +- .../sql/calcite/schema/SystemSchema.java | 182 ++++++------------ .../org/apache/druid/sql/guice/SqlModule.java | 1 - .../schema/DruidCalciteSchemaModuleTest.java | 3 + .../sql/calcite/schema/SystemSchemaTest.java | 99 ++++++++-- .../druid/sql/calcite/util/CalciteTests.java | 8 +- .../apache/druid/sql/guice/SqlModuleTest.java | 3 + 22 files changed, 420 insertions(+), 244 deletions(-) rename server/src/main/java/org/apache/druid/segment/metadata/{DataSourceSchema.java => DataSourceInformation.java} (91%) diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index a440a36923e6..b550a9c8d421 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -54,27 +54,25 @@ public class SegmentStatusInCluster implements Comparable getSegmentLoadInfos(); - - } diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index 66b739cf51cb..b8f384ae1ba7 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -22,7 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; @@ -49,7 +49,7 @@ public interface CoordinatorClient /** * Fetches schema for the given dataSources. */ - ListenableFuture> fetchDataSourceSchema(Set datasources); + ListenableFuture> fetchDataSourceInformation(Set datasources); /** * Returns a new instance backed by a ServiceClient which follows the provided retryPolicy diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index 24e6968ad6ab..f11c3de2eaec 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -30,7 +30,7 @@ import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; import org.apache.druid.rpc.ServiceRetryPolicy; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.timeline.DataSegment; import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; @@ -110,9 +110,9 @@ public ListenableFuture> fetchUsedSegments(String dataSource, } @Override - public ListenableFuture> fetchDataSourceSchema(Set dataSources) + public ListenableFuture> fetchDataSourceInformation(Set dataSources) { - final String path = "/druid/coordinator/v1/metadata/dataSourceSchema"; + final String path = "/druid/coordinator/v1/metadata/dataSourceInformation"; if (null == dataSources) { dataSources = new HashSet<>(); @@ -123,7 +123,7 @@ public ListenableFuture> fetchDataSourceSchema(Set JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) + holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) ); } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java similarity index 91% rename from server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java rename to server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index 0528cfe3b1f8..6501704bddac 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -9,13 +9,13 @@ /** * Encapsulates schema information of a dataSource. */ -public class DataSourceSchema +public class DataSourceInformation { private final String datasource; private final RowSignature rowSignature; @JsonCreator - public DataSourceSchema( + public DataSourceInformation( @JsonProperty("datasource") String datasource, @JsonProperty("rowSignature") RowSignature rowSignature) { @@ -44,7 +44,7 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - DataSourceSchema that = (DataSourceSchema) o; + DataSourceInformation that = (DataSourceInformation) o; return Objects.equals(datasource, that.datasource) && Objects.equals( rowSignature, that.rowSignature diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 56328fbc3dec..5077ed046577 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -124,7 +124,7 @@ public class SegmentMetadataCache * Map of DataSource -> DruidTable. * This map can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final ConcurrentMap tables = new ConcurrentHashMap<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -201,7 +201,7 @@ public class SegmentMetadataCache * Currently, there are 2 threads that can access these variables. * * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceSchema} if necessary + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceInformation} if necessary * based on the information collected via timeline callbacks. */ protected final Object lock = new Object(); @@ -428,13 +428,13 @@ public void refresh(final Set segmentsToRefresh, final Set da public void rebuildDatasource(String dataSource) { - final DataSourceSchema druidTable = buildDruidTable(dataSource); + final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final DataSourceSchema oldTable = tables.put(dataSource, druidTable); + final DataSourceInformation oldTable = tables.put(dataSource, druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -454,12 +454,12 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } - public DataSourceSchema getDatasource(String name) + public DataSourceInformation getDatasource(String name) { return tables.get(name); } - public Map getDataSourceSchemaMap() + public Map getDataSourceSchemaMap() { return ImmutableMap.copyOf(tables); } @@ -818,7 +818,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin @VisibleForTesting @Nullable - public DataSourceSchema buildDruidTable(final String dataSource) + public DataSourceInformation buildDruidTable(final String dataSource) { ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); @@ -846,7 +846,7 @@ public DataSourceSchema buildDruidTable(final String dataSource) final RowSignature.Builder builder = RowSignature.builder(); columnTypes.forEach(builder::add); - return new DataSourceSchema(dataSource, builder.build()); + return new DataSourceInformation(dataSource, builder.build()); } public Map getSegmentMetadataSnapshot() diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 1de89a11de73..92e5dd03ba40 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -31,7 +31,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.JettyUtils; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -201,6 +201,9 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .map(segment -> { // The replication factor for unloaded segments is 0 as they will be unloaded soon boolean isOvershadowed = overshadowedSegments.contains(segment); + Integer replicationFactor = isOvershadowed ? (Integer) 0 + : coordinator.getReplicationFactor(segment.getId()); + Long numRows = null; if (null != segmentMetadataCache) { AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( @@ -211,29 +214,34 @@ private Response getAllUsedSegmentsWithAdditionalDetails( numRows = availableSegmentMetadata.getNumRows(); } } - Integer replicationFactor = isOvershadowed ? (Integer) 0 - : coordinator.getReplicationFactor(segment.getId()); segmentAlreadySeen.add(segment.getId()); - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, numRows, true); + return new SegmentStatusInCluster( + segment, + isOvershadowed, + replicationFactor, + numRows, + // published segment can't be realtime + false); }); Stream finalSegments = segmentStatus; - if (null != includeRealtimeSegments && segmentMetadataCache != null) { - log.info("printing the content in smc cache %s", segmentMetadataCache.getSegmentMetadataSnapshot().values()); + if (null != includeRealtimeSegments && null != segmentMetadataCache) { final Stream realtimeSegmentStatus = segmentMetadataCache .getSegmentMetadataSnapshot() .values() .stream() - .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment() - .getId())) - .map(availableSegmentMetadata -> new SegmentStatusInCluster( - availableSegmentMetadata.getSegment(), - false, - (int) availableSegmentMetadata.getNumReplicas(), - availableSegmentMetadata.getNumRows(), - availableSegmentMetadata.isRealtime() != 0 - )); + .filter(availableSegmentMetadata -> + !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment().getId())) + .map(availableSegmentMetadata -> + new SegmentStatusInCluster( + availableSegmentMetadata.getSegment(), + false, + // replication factor is null for unpublished segments + null, + availableSegmentMetadata.getNumRows(), + availableSegmentMetadata.isRealtime() != 0 + )); finalSegments = Stream.concat(segmentStatus, realtimeSegmentStatus); } @@ -351,21 +359,21 @@ public Response getUsedSegment( * Return schema for the given dataSources. */ @POST - @Path("/dataSourceSchema") + @Path("/dataSourceInformation") @Produces(MediaType.APPLICATION_JSON) - public Response getDataSourceSchema( + public Response getDataSourceInformation( List dataSources ) { if (null == segmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } - Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceSchemaMap(); + Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceSchemaMap(); - List results = new ArrayList<>(); + List results = new ArrayList<>(); List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; - for (Map.Entry entry : dataSourceSchemaMap.entrySet()) { + for (Map.Entry entry : dataSourceSchemaMap.entrySet()) { if (dataSourcesToRetain.contains(entry.getKey())) { results.add(entry.getValue()); } diff --git a/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java b/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java index b2cde828092e..bbceeb4523e1 100644 --- a/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java +++ b/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java @@ -22,7 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; @@ -50,7 +50,7 @@ public ListenableFuture> fetchUsedSegments(String dataSource, } @Override - public ListenableFuture> fetchDataSourceSchema(Set datasources) + public ListenableFuture> fetchDataSourceInformation(Set datasources) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 92ad857476e1..60421be19810 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -138,7 +138,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV ) { @Override - public DataSourceSchema buildDruidTable(final String dataSource) + public DataSourceInformation buildDruidTable(final String dataSource) { doInLock(() -> { try { @@ -246,7 +246,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() ) { @Override - public DataSourceSchema buildDruidTable(final String dataSource) + public DataSourceInformation buildDruidTable(final String dataSource) { doInLock(() -> { try { diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java index 16567719dffc..cf0511dd6f8a 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java @@ -51,7 +51,7 @@ public void testDefaultConfig() final Properties properties = new Properties(); provider.inject(properties, injector.getInstance(JsonConfigurator.class)); final SegmentMetadataCacheConfig config = provider.get(); - Assert.assertTrue(config.isAwaitInitializationOnStart()); + Assert.assertFalse(config.isAwaitInitializationOnStart()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index dc18cfb79741..aa9df38b9b71 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -288,9 +288,9 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheCon ) { @Override - public DataSourceSchema buildDruidTable(String dataSource) + public DataSourceInformation buildDruidTable(String dataSource) { - DataSourceSchema table = super.buildDruidTable(dataSource); + DataSourceInformation table = super.buildDruidTable(dataSource); buildTableLatch.countDown(); return table; } @@ -371,14 +371,14 @@ public void testSchemaInit() throws InterruptedException public void testGetTableMapFoo() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DataSourceSchema fooDs = schema.getDatasource("foo"); + final DataSourceInformation fooDs = schema.getDatasource("foo"); } @Test public void testGetTableMapFoo2() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DataSourceSchema fooDs = schema.getDatasource("foo2"); + final DataSourceInformation fooDs = schema.getDatasource("foo2"); } @Test @@ -395,7 +395,7 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli } } ); - final DataSourceSchema fooDs = schema.getDatasource(SOME_DATASOURCE); + final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); } @@ -405,7 +405,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DataSourceSchema fooDs = schema.getDatasource(SOME_DATASOURCE); + final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); } @@ -1002,7 +1002,7 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte { SegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DataSourceSchema fooTable = schema3.getDatasource("foo"); + DataSourceInformation fooTable = schema3.getDatasource("foo"); Assert.assertNotNull(fooTable); markDataSourceLatch = new CountDownLatch(1); @@ -1052,7 +1052,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw { SegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DataSourceSchema fooTable = schema.getDatasource("foo"); + DataSourceInformation fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); markDataSourceLatch = new CountDownLatch(1); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index c0978e80c367..589d2a3ae52f 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -19,21 +19,26 @@ package org.apache.druid.server.http; +import com.google.api.client.util.Sets; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.junit.Assert; import org.junit.Before; @@ -42,22 +47,29 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; public class MetadataResourceTest { private static final String DATASOURCE1 = "datasource1"; - private MetadataResource metadataResource; - private HttpServletRequest request; - private final DataSegment[] segments = CreateDataSegments.ofDatasource(DATASOURCE1) .forIntervals(3, Granularities.DAY) .withNumPartitions(2) .eachOfSizeInMb(500) .toArray(new DataSegment[0]); - + private HttpServletRequest request; + private SegmentsMetadataManager segmentsMetadataManager; + private IndexerMetadataStorageCoordinator storageCoordinator; + private DruidCoordinator coordinator; + + + private MetadataResource metadataResource; + @Before public void setUp() { @@ -65,7 +77,7 @@ public void setUp() Mockito.doReturn(Mockito.mock(AuthenticationResult.class)) .when(request).getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT); - SegmentsMetadataManager segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class); + segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class); ImmutableDruidDataSource druidDataSource1 = new ImmutableDruidDataSource( DATASOURCE1, ImmutableMap.of(), @@ -81,7 +93,7 @@ public void setUp() .when(segmentsMetadataManager) .getImmutableDataSourceWithUsedSegments(DATASOURCE1); - DruidCoordinator coordinator = Mockito.mock(DruidCoordinator.class); + coordinator = Mockito.mock(DruidCoordinator.class); Mockito.doReturn(2).when(coordinator).getReplicationFactor(segments[0].getId()); Mockito.doReturn(null).when(coordinator).getReplicationFactor(segments[1].getId()); Mockito.doReturn(1).when(coordinator).getReplicationFactor(segments[2].getId()); @@ -89,7 +101,7 @@ public void setUp() Mockito.doReturn(ImmutableSet.of(segments[3])) .when(dataSourcesSnapshot).getOvershadowedSegments(); - IndexerMetadataStorageCoordinator storageCoordinator = Mockito.mock(IndexerMetadataStorageCoordinator.class); + storageCoordinator = Mockito.mock(IndexerMetadataStorageCoordinator.class); Mockito.doReturn(segments[4]) .when(storageCoordinator) .retrieveUsedSegmentForId(segments[4].getId().toString()); @@ -113,11 +125,110 @@ public void testGetAllSegmentsWithOvershadowedStatus() final List resultList = extractSegmentStatusList(response); Assert.assertEquals(resultList.size(), 4); - Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 5L, true), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 5L, true), resultList.get(1)); - Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, 5L, true), resultList.get(2)); + Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, null, false), resultList.get(0)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, null, false), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, null, false), resultList.get(2)); + // Replication factor should be 0 as the segment is overshadowed + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); + } + + @Test + public void testGetAllSegmentsIncludingRealtime() + { + SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); + + String dataSource2 = "datasource2"; + + DataSegment[] realTimeSegments = + CreateDataSegments.ofDatasource(dataSource2) + .forIntervals(3, Granularities.DAY) + .withNumPartitions(2) + .eachOfSizeInMb(500) + .toArray(new DataSegment[0]); + + Mockito.doReturn(null).when(coordinator).getReplicationFactor(realTimeSegments[0].getId()); + Mockito.doReturn(null).when(coordinator).getReplicationFactor(realTimeSegments[1].getId()); + Map availableSegments = new HashMap<>(); + availableSegments.put( + segments[0].getId(), + AvailableSegmentMetadata.builder( + segments[0], + 0L, + Sets.newHashSet(), + null, + 20L + ).build() + ); + availableSegments.put( + segments[1].getId(), + AvailableSegmentMetadata.builder( + segments[1], + 0L, + Sets.newHashSet(), + null, + 30L + ).build() + ); + availableSegments.put( + segments[1].getId(), + AvailableSegmentMetadata.builder( + segments[1], + 0L, + Sets.newHashSet(), + null, + 30L + ).build() + ); + availableSegments.put( + realTimeSegments[0].getId(), + AvailableSegmentMetadata.builder( + realTimeSegments[0], + 1L, + Sets.newHashSet(), + null, + 10L + ).build() + ); + availableSegments.put( + realTimeSegments[1].getId(), + AvailableSegmentMetadata.builder( + realTimeSegments[1], + 1L, + Sets.newHashSet(), + null, + 40L + ).build() + ); + + Mockito.doReturn(availableSegments).when(segmentMetadataCache).getSegmentMetadataSnapshot(); + + Mockito.doReturn(availableSegments.get(segments[0].getId())) + .when(segmentMetadataCache) + .getAvailableSegmentMetadata(DATASOURCE1, segments[0].getId()); + + Mockito.doReturn(availableSegments.get(segments[1].getId())) + .when(segmentMetadataCache) + .getAvailableSegmentMetadata(DATASOURCE1, segments[1].getId()); + + metadataResource = new MetadataResource( + segmentsMetadataManager, + storageCoordinator, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + coordinator, + segmentMetadataCache + ); + + Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", "includeRealtimeSegments"); + + final List resultList = extractSegmentStatusList(response); + Assert.assertEquals(resultList.size(), 6); + Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 20L, false), resultList.get(0)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 30L, false), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, null, false), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, 5L, true), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[0], false, null, 10L, true), resultList.get(4)); + Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[1], false, null, 40L, true), resultList.get(5)); } @Test diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 9fa0f3938b81..0a0428ad1d50 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -28,9 +28,8 @@ import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable; @@ -92,13 +91,10 @@ public void refresh(final Set segmentsToRefresh, final Set da // Fetch dataSource schema from the Coordinator try { - FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceSchema(dataSourcesToQuery), true) + FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) .forEach(item -> polledDataSourceSchema.put( item.getDatasource(), - physicalDatasourceMetadataBuilder.build( - item.getDatasource(), - item.getRowSignature() - ) + physicalDatasourceMetadataBuilder.build(item) )); } catch (Exception e) { log.error("Exception querying coordinator for schema"); @@ -140,14 +136,13 @@ public void refresh(final Set segmentsToRefresh, final Set da @Override public void rebuildDatasource(String dataSource) { - final DataSourceSchema druidTable = buildDruidTable(dataSource); + final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final DatasourceTable.PhysicalDatasourceMetadata physicalDatasourceMetadata = - physicalDatasourceMetadataBuilder.build(dataSource, druidTable.getRowSignature()); + final DatasourceTable.PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); final DatasourceTable.PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index 9ef381b10b3f..38b570b779d1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -28,11 +28,9 @@ import com.google.common.util.concurrent.Uninterruptibles; import com.google.inject.Inject; import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.DataSegmentInterner; import org.apache.druid.client.JsonParserIterator; import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.client.selector.ServerSelector; import org.apache.druid.concurrent.LifecycleLock; import org.apache.druid.discovery.DruidLeaderClient; import org.apache.druid.guice.ManageLifecycle; @@ -74,11 +72,8 @@ public class BrokerSegmentMetadataView private final BrokerSegmentWatcherConfig segmentWatcherConfig; private final DruidLeaderClient druidLeaderClient; private final ObjectMapper objectMapper; - - private final boolean isMetadataSegmentCacheEnabled; - /** * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and * sys.segments queries return the segments in sorted order based on segmentId. @@ -161,14 +156,16 @@ public void stop() protected Iterator getSegmentTableView() { + // set of published and realtime segments final ImmutableSortedSet segments = getSegmentMetadata(); + + // set of available segments and metadata final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); final List segmentsTableViews = new ArrayList<>(); Set seenSegments = new HashSet<>(); - for (SegmentStatusInCluster segmentStatusInCluster : segments) - { + for (SegmentStatusInCluster segmentStatusInCluster : segments) { DataSegment segment = segmentStatusInCluster.getDataSegment(); SegmentId segmentId = segment.getId(); AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataMap.get(segmentId); @@ -188,6 +185,12 @@ protected Iterator getSegmentTableView() long isRealtime = Boolean.TRUE.equals(segmentStatusInCluster.isRealtime()) ? 1 : 0; + // set of segments returned from coordinator include published and realtime segments + // so realtime segments are not published and vice versa + boolean isPublished = !segmentStatusInCluster.isRealtime(); + + // is_active is true for published segments that are not overshadowed + boolean isActive = isPublished && !segmentStatusInCluster.isOvershadowed(); SegmentTableView segmentTableView = new SegmentTableView( segment, @@ -195,28 +198,48 @@ protected Iterator getSegmentTableView() isRealtime, numReplicas, numRows, + // If the segment is unpublished, we won't have this information yet. + // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. + // This should be automatically updated in the next refesh with Coordinator. segmentStatusInCluster.getReplicationFactor(), - segmentStatusInCluster.isOvershadowed() + // If the segment is unpublished this value would be false + segmentStatusInCluster.isOvershadowed(), + isPublished, + isActive ); + seenSegments.add(segmentId); segmentsTableViews.add((segmentTableView)); } - for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) - { + for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) { if (seenSegments.contains(availableSegmentMetadataEntry.getKey())) { continue; } AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataEntry.getValue(); + + // since all published segments would have been covered in the previous loop + // the curent set of segments must be unpublished + boolean isPublished = false; + + // is_active is true for unpublished segments iff they are realtime + boolean isActive = availableSegmentMetadata.isRealtime() == 1L; + SegmentTableView segmentTableView = new SegmentTableView( availableSegmentMetadata.getSegment(), + // segments announced by historicals or realtime tasks are assumed to be available 1L, availableSegmentMetadata.isRealtime(), availableSegmentMetadata.getNumReplicas(), availableSegmentMetadata.getNumRows(), + // If the segment is unpublished, we won't have this information yet. null, - false + // there is an assumption here that unpublished segments are never overshadowed + false, + isPublished, + isActive ); + segmentsTableViews.add(segmentTableView); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index 5dfbfb664440..5b344621f1f8 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -24,6 +24,7 @@ import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; import org.apache.druid.sql.calcite.table.DatasourceTable; @@ -39,8 +40,10 @@ public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, Segmen this.segmentManager = segmentManager; } - DatasourceTable.PhysicalDatasourceMetadata build(String dataSource, RowSignature rowSignature) + DatasourceTable.PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) { + final String dataSource = dataSourceInformation.getDatasource(); + final RowSignature rowSignature = dataSourceInformation.getRowSignature(); final TableDataSource tableDataSource; // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 28e322a3ddca..afa0986467d5 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -27,10 +27,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; import com.google.common.util.concurrent.Futures; import com.google.inject.Inject; @@ -72,7 +68,6 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthenticationResult; @@ -85,7 +80,6 @@ import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.SegmentStatusInCluster; import org.jboss.netty.handler.codec.http.HttpMethod; import javax.annotation.Nullable; @@ -100,7 +94,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; public class SystemSchema extends AbstractSchema @@ -122,20 +115,6 @@ public class SystemSchema extends AbstractSchema segment.getDataSource()) ); - private static final long REPLICATION_FACTOR_UNKNOWN = -1L; - - /** - * Booleans constants represented as long type, - * where 1 = true and 0 = false to make it easy to count number of segments - * which are published, available etc. - */ - private static final long IS_ACTIVE_FALSE = 0L; - private static final long IS_ACTIVE_TRUE = 1L; - private static final long IS_PUBLISHED_FALSE = 0L; - private static final long IS_PUBLISHED_TRUE = 1L; - private static final long IS_OVERSHADOWED_FALSE = 0L; - private static final long IS_OVERSHADOWED_TRUE = 1L; - static final RowSignature SEGMENTS_SIGNATURE = RowSignature .builder() .add("segment_id", ColumnType.STRING) @@ -291,22 +270,18 @@ public Enumerable scan(DataContext root) (long) segment.getShardSpec().getPartitionNum(), val.getNumReplicas(), val.getNumRows(), - //is_active is true for published segments that are not overshadowed - val.isOvershadowed() ? IS_ACTIVE_FALSE : IS_ACTIVE_TRUE, - //is_published is true for published segments - (val.getIsRealtime() == 0) ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, - val.getIsAvailable(), - val.getIsRealtime(), - val.isOvershadowed() ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE, + val.isActive(), + val.isPublished(), + val.isAvailable(), + val.isRealtime(), + val.isOvershadowed(), segment.getShardSpec() == null ? null : jsonMapper.writeValueAsString(segment.getShardSpec()), segment.getDimensions() == null ? null : jsonMapper.writeValueAsString(segment.getDimensions()), segment.getMetrics() == null ? null : jsonMapper.writeValueAsString(segment.getMetrics()), segment.getLastCompactionState() == null ? null : jsonMapper.writeValueAsString(segment.getLastCompactionState()), - // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. - // This should be automatically updated in the next refesh with Coordinator. - val.getReplicationFactor() == null ? REPLICATION_FACTOR_UNKNOWN : (long) val.getReplicationFactor() + val.getReplicationFactor() }; } catch (JsonProcessingException e) { @@ -366,31 +341,50 @@ private Iterator> getAuthorizedAvaila protected static class SegmentTableView { + /** + * Booleans constants represented as long type, + * where 1 = true and 0 = false to make it easy to count number of segments + * which are published, available etc. + */ + private static final long IS_ACTIVE_FALSE = 0L; + private static final long IS_ACTIVE_TRUE = 1L; + private static final long IS_PUBLISHED_FALSE = 0L; + private static final long IS_PUBLISHED_TRUE = 1L; + private static final long IS_OVERSHADOWED_FALSE = 0L; + private static final long IS_OVERSHADOWED_TRUE = 1L; + private static final long REPLICATION_FACTOR_UNKNOWN = -1L; + private final DataSegment segment; - private final long isAvailable; - private final long isRealtime; + private final long available; + private final long realtime; private final long numReplicas; private final long numRows; - private final Integer replicationFactor; - private final boolean isOvershadowed; + private final long replicationFactor; + private final long overshadowed; + private final long published; + private final long active; public SegmentTableView( DataSegment segment, long isAvailable, - long isRealtime, + long realtime, long numReplicas, long numRows, Integer replicationFactor, - boolean isOvershadowed + boolean overshadowed, + boolean isPublished, + boolean active ) { this.segment = segment; - this.isAvailable = isAvailable; - this.isRealtime = isRealtime; + this.available = isAvailable; + this.realtime = realtime; this.numReplicas = numReplicas; this.numRows = numRows; - this.replicationFactor = replicationFactor; - this.isOvershadowed = isOvershadowed; + this.replicationFactor = (null == replicationFactor) ? REPLICATION_FACTOR_UNKNOWN : (long) replicationFactor; + this.overshadowed = overshadowed ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE; + this.published = isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE; + this.active = active ? IS_ACTIVE_TRUE : IS_ACTIVE_FALSE; } public DataSegment getSegment() @@ -398,14 +392,14 @@ public DataSegment getSegment() return segment; } - public long getIsAvailable() + public long isAvailable() { - return isAvailable; + return available; } - public long getIsRealtime() + public long isRealtime() { - return isRealtime; + return realtime; } public long getNumReplicas() @@ -418,14 +412,24 @@ public long getNumRows() return numRows; } - public Integer getReplicationFactor() + public long getReplicationFactor() { return replicationFactor; } - public boolean isOvershadowed() + public long isOvershadowed() { - return isOvershadowed; + return overshadowed; + } + + public long isPublished() + { + return published; + } + + public long isActive() + { + return active; } @Override @@ -433,90 +437,16 @@ public String toString() { return "SegmentTableView{" + "segmentId=" + segment.getId() + - ", isAvailable=" + isAvailable + - ", isRealtime=" + isRealtime + + ", isAvailable=" + available + + ", isRealtime=" + realtime + ", numReplicas=" + numReplicas + ", numRows=" + numRows + ", replicationFactor=" + replicationFactor + - ", isOvershadowed=" + isOvershadowed + + ", isOvershadowed=" + overshadowed + + ", isPublished=" + published + '}'; } } - - protected static class SegmentTableView1 - { - // pass the stitched - private final Map availableSegmentMetadata; - private final Iterator publishedSegments; // coordinator - - private final int totalSegmentsCount; - - public SegmentTableView1( - Map availableSegmentMetadata, - ImmutableSortedSet publishedSegments - ) - { - this.availableSegmentMetadata = availableSegmentMetadata; - this.publishedSegments = publishedSegments.iterator(); - this.totalSegmentsCount = availableSegmentMetadata.size(); - } - - public Map getAvailableSegmentMetadata() - { - return availableSegmentMetadata; - } - - public Iterator getPublishedSegments() - { - return publishedSegments; - } - - public int totalSegmentCount() - { - return totalSegmentsCount; - } - } - - protected static class PartialSegmentData - { - private final long isAvailable; - private final long isRealtime; - private final long numReplicas; - private final long numRows; - - public PartialSegmentData( - final long isAvailable, - final long isRealtime, - final long numReplicas, - final long numRows - ) - { - this.isAvailable = isAvailable; - this.isRealtime = isRealtime; - this.numReplicas = numReplicas; - this.numRows = numRows; - } - - public long isAvailable() - { - return isAvailable; - } - - public long isRealtime() - { - return isRealtime; - } - - public long getNumReplicas() - { - return numReplicas; - } - - public long getNumRows() - { - return numRows; - } - } } /** diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 56d0d2d5d41f..63ef71585dd9 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -62,7 +62,6 @@ public class SqlModule implements Module public static final String PROPERTY_SQL_VIEW_MANAGER_TYPE = "druid.sql.viewmanager.type"; public static final String PROPERTY_SQL_SCHEMA_MANAGER_TYPE = "druid.sql.schemamanager.type"; public static final String PROPERTY_SQL_APPROX_COUNT_DISTINCT_CHOICE = "druid.sql.approxCountDistinct.function"; - private Properties props; @Inject diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModuleTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModuleTest.java index 1d646214ec74..0b5fb609fa22 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModuleTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModuleTest.java @@ -31,6 +31,8 @@ import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.client.indexing.IndexingService; import org.apache.druid.client.indexing.NoopOverlordClient; import org.apache.druid.discovery.DruidLeaderClient; @@ -132,6 +134,7 @@ public void setUp() binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); binder.bind(ServiceEmitter.class).toInstance(new ServiceEmitter("", "", null)); binder.bind(OverlordClient.class).to(NoopOverlordClient.class); + binder.bind(CoordinatorClient.class).to(NoopCoordinatorClient.class); }, new LifecycleModule(), target); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index b17eb7713688..a159dcc24e5c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -19,6 +19,7 @@ package org.apache.druid.sql.calcite.schema; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; @@ -36,6 +37,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; @@ -62,6 +64,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.response.FullResponseHolder; import org.apache.druid.java.util.http.client.response.HttpResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; @@ -106,9 +109,14 @@ import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.SegmentStatusInCluster; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; +import org.jboss.netty.handler.codec.http.DefaultHttpResponse; +import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.handler.codec.http.HttpVersion; import org.joda.time.DateTime; import org.junit.AfterClass; import org.junit.Assert; @@ -122,6 +130,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -153,7 +162,6 @@ public class SystemSchemaTest extends CalciteTestBase ); private SystemSchema schema; - private SpecificSegmentsQuerySegmentWalker walker; private DruidLeaderClient client; private DruidLeaderClient coordinatorClient; private OverlordClient overlordClient; @@ -162,7 +170,6 @@ public class SystemSchemaTest extends CalciteTestBase private StringFullResponseHolder responseHolder; private BytesAccumulatingResponseHandler responseHandler; private Request request; - private DruidSchema druidSchema; private AuthorizerMapper authMapper; private static QueryRunnerFactoryConglomerate conglomerate; private static Closer resourceCloser; @@ -247,7 +254,7 @@ public void setUp() throws Exception .rows(ROWS3) .buildMMappedIndex(); - walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) + SpecificSegmentsQuerySegmentWalker walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) .add(segment1, index1) .add(segment2, index2) .add(segment3, index3); @@ -265,10 +272,69 @@ public void setUp() throws Exception ), new NoopCoordinatorClient() ); + cache.start(); cache.awaitInitialization(); - druidSchema = new DruidSchema(cache, null); - metadataView = EasyMock.createMock(BrokerSegmentMetadataView.class); + + DruidNode coordinatorNode = new DruidNode("test-coordinator", "localhost", false, 8081, null, true, false); + DruidLeaderClient druidLeaderClient = new DruidLeaderClient( + new CalciteTests.FakeHttpClient(), + new CalciteTests.FakeDruidNodeDiscoveryProvider( + ImmutableMap.of( + NodeRole.COORDINATOR, + new CalciteTests.FakeDruidNodeDiscovery(ImmutableMap.of(NodeRole.COORDINATOR, coordinatorNode)) + ) + ), + NodeRole.COORDINATOR, + "/simple/leader" + ) + { + @Override + public String findCurrentLeader() + { + return coordinatorNode.getHostAndPortToUse(); + } + + @Override + public Request makeRequest(HttpMethod httpMethod, String urlPath) throws IOException + { + return new Request(httpMethod, new URL(StringUtils.format("http://%s%s", coordinatorNode.getHostAndPortToUse(), urlPath))); + } + + @Override + public > H go(Request request, HttpResponseHandler responseHandler) + throws IOException + { + List segmentStatusInClusterList = new ArrayList<>(Arrays.asList( + new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 0L, false), + new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 0L, false), + new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 0L, false), + new SegmentStatusInCluster(segment1, true, 2, 3L, false), + new SegmentStatusInCluster(segment2, false, 0, 3L, false) + )); + + InputStreamFullResponseHolder responseHolder = new InputStreamFullResponseHolder(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK)); + byte[] bytesToWrite = mapper.writeValueAsString(segmentStatusInClusterList).getBytes(StandardCharsets.UTF_8); + responseHolder.addChunk(bytesToWrite); + responseHolder.done(); + return (H) responseHolder; + } + }; + + metadataView = new BrokerSegmentMetadataView( + druidLeaderClient, + mapper, + new BrokerSegmentWatcherConfig(), + new BrokerSegmentMetadataCacheConfig() { + @Override + public boolean isMetadataSegmentCacheEnable() + { + return false; + } + }, + cache + ); + druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class); schema = new SystemSchema( @@ -283,6 +349,17 @@ public void setUp() throws Exception ); } + @Test + public void test1() throws IOException + { + SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster(segment1, true, 2, 4L, Boolean.TRUE); + byte[] x = mapper.writeValueAsBytes(segmentStatusInCluster); + SegmentStatusInCluster w = mapper.readValue(x, new TypeReference() + { + }); + int p = 1; + } + private final CompactionState expectedCompactionState = new CompactionState( new DynamicPartitionsSpec(null, null), null, @@ -570,17 +647,7 @@ public void testSegmentsTable() throws Exception { final SegmentsTable segmentsTable = new SegmentsTable(metadataView, new ObjectMapper(), authMapper); - final Set segmentTableViews = new HashSet<>(Arrays.asList( - new SegmentTableView(publishedCompactedSegment1, 1L, 0L, 1, 2, 2, true), - new SegmentTableView(publishedCompactedSegment2, 1L, 0L, 0, 2, 0, false), - new SegmentTableView(publishedUncompactedSegment3, 1L, 0L, 1, 2, 2, false), - new SegmentTableView(segment1, 1L, 0L, 1, 2, 2, true), - new SegmentTableView(segment2, 1L, 0L, 1, 2, 0, false) - )); - - EasyMock.expect(metadataView.getSegmentTableView()).andReturn(segmentTableViews.iterator()).once(); - - EasyMock.replay(client, request, responseHolder, responseHandler, metadataView); + EasyMock.replay(client, request, responseHolder, responseHandler); DataContext dataContext = createDataContext(Users.SUPER); final List rows = segmentsTable.scan(dataContext).toList(); rows.sort((Object[] row1, Object[] row2) -> ((Comparable) row1[0]).compareTo(row2[0])); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index 3f5e9b14f5e9..66457b292f83 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -433,7 +433,7 @@ public static DruidSchemaCatalog createMockRootSchema( /** * A fake {@link HttpClient} for {@link #createMockSystemSchema}. */ - private static class FakeHttpClient implements HttpClient + public static class FakeHttpClient implements HttpClient { @Override public ListenableFuture go( @@ -458,7 +458,7 @@ public ListenableFuture go( /** * A fake {@link DruidNodeDiscoveryProvider} for {@link #createMockSystemSchema}. */ - private static class FakeDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider + public static class FakeDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider { private final Map nodeDiscoveries; @@ -484,7 +484,7 @@ public DruidNodeDiscovery getForNodeRole(NodeRole nodeRole) } } - private static class FakeDruidNodeDiscovery implements DruidNodeDiscovery + public static class FakeDruidNodeDiscovery implements DruidNodeDiscovery { private final Set nodes; @@ -493,7 +493,7 @@ private static class FakeDruidNodeDiscovery implements DruidNodeDiscovery this.nodes = new HashSet<>(); } - FakeDruidNodeDiscovery(Map nodes) + public FakeDruidNodeDiscovery(Map nodes) { this.nodes = Sets.newHashSetWithExpectedSize(nodes.size()); nodes.forEach((k, v) -> { diff --git a/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java b/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java index 76abef005422..dfd366250457 100644 --- a/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java +++ b/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java @@ -31,6 +31,8 @@ import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.client.indexing.IndexingService; import org.apache.druid.client.indexing.NoopOverlordClient; import org.apache.druid.discovery.DruidLeaderClient; @@ -209,6 +211,7 @@ private Injector makeInjectorWithProperties(final Properties props) binder.bind(ResponseContextConfig.class).toInstance(SqlResourceTest.TEST_RESPONSE_CONTEXT_CONFIG); binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); binder.bind(OverlordClient.class).to(NoopOverlordClient.class); + binder.bind(CoordinatorClient.class).to(NoopCoordinatorClient.class); }, sqlModule, new TestViewManagerModule() From bc8396c0409b9c971cf7f5e4472de6fa431692f4 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 8 Sep 2023 22:12:49 +0530 Subject: [PATCH 11/82] minor change --- .../client/coordinator/CoordinatorClient.java | 2 +- .../metadata/SegmentMetadataCacheConfig.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index b6a966c408b0..2f51ca2ec743 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -48,7 +48,7 @@ public interface CoordinatorClient ListenableFuture> fetchUsedSegments(String dataSource, List intervals); /** - * Fetches schema for the given dataSources. + * Fetches information for the given dataSources. */ ListenableFuture> fetchDataSourceInformation(Set datasources); diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index b1932485b1ad..4cf8d0fb0019 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.segment.metadata; import com.fasterxml.jackson.annotation.JsonProperty; From fbab4c88ac7e6f4e83a808fb9076fe7d16897a88 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 8 Sep 2023 23:07:39 +0530 Subject: [PATCH 12/82] Remove logic to refactor sys segments table building logic --- .../metadata/SegmentMetadataCache.java | 2 +- .../schema/BrokerSegmentMetadataView.java | 355 ------------------ .../druid/sql/calcite/schema/DruidSchema.java | 5 + .../calcite/schema/MetadataSegmentView.java | 244 ++++++++++++ .../sql/calcite/schema/SystemSchema.java | 257 +++++++------ .../org/apache/druid/sql/guice/SqlModule.java | 1 + .../sql/calcite/schema/SystemSchemaTest.java | 111 ++---- .../druid/sql/calcite/util/CalciteTests.java | 47 +-- .../sql/calcite/util/QueryFrameworkUtils.java | 2 +- 9 files changed, 435 insertions(+), 589 deletions(-) delete mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 5483571ce59b..32feb8844179 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -871,7 +871,7 @@ public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, S * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. * As a result, the returned value might be inexact. */ - int getTotalSegments() + public int getTotalSegments() { return totalSegments; } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java deleted file mode 100644 index 38b570b779d1..000000000000 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.sql.calcite.schema; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.util.concurrent.Uninterruptibles; -import com.google.inject.Inject; -import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.DataSegmentInterner; -import org.apache.druid.client.JsonParserIterator; -import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.concurrent.LifecycleLock; -import org.apache.druid.discovery.DruidLeaderClient; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.lifecycle.LifecycleStop; -import org.apache.druid.java.util.emitter.EmittingLogger; -import org.apache.druid.metadata.SegmentsMetadataManager; -import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.SegmentStatusInCluster; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * This class polls the Coordinator in background to keep the latest published segments. - * Provides {@link #getSegmentTableView()} for sys segments table view. - * - * This class polls the data from {@link SegmentsMetadataManager} object in the memory of the - * currently leading Coordinator via HTTP queries. - */ -@ManageLifecycle -public class BrokerSegmentMetadataView -{ - private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataView.class); - - private final BrokerSegmentWatcherConfig segmentWatcherConfig; - private final DruidLeaderClient druidLeaderClient; - private final ObjectMapper objectMapper; - private final boolean isMetadataSegmentCacheEnabled; - - /** - * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and - * sys.segments queries return the segments in sorted order based on segmentId. - * - * Volatile since this reference is reassigned in {@code pollSegmentMetadata()} - * and then read in {@code getSegmentMetadata()} - * from other threads. - */ - @MonotonicNonNull - private volatile ImmutableSortedSet segmentMetadata = null; - - private final BrokerSegmentMetadataCache segmentMetadataCache; - - /** - * Caches the replication factor for segment IDs. In case of coordinator restarts or leadership re-elections, - * the coordinator API returns `null` replication factor until load rules are evaluated. - * The cache can be used during these periods to continue serving the previously fetched values. - */ - private final Cache segmentIdToReplicationFactor; - private final ScheduledExecutorService scheduledExec; - private final long pollPeriodInMS; - private final LifecycleLock lifecycleLock = new LifecycleLock(); - private final CountDownLatch segmentMetadataCachePopulated = new CountDownLatch(1); - - @Inject - public BrokerSegmentMetadataView( - final @Coordinator DruidLeaderClient druidLeaderClient, - final ObjectMapper objectMapper, - final BrokerSegmentWatcherConfig segmentWatcherConfig, - final BrokerSegmentMetadataCacheConfig config, - final BrokerSegmentMetadataCache segmentMetadataCache - ) - { - Preconditions.checkNotNull(config, "BrokerSegmentMetadataCacheConfig"); - this.druidLeaderClient = druidLeaderClient; - this.objectMapper = objectMapper; - this.segmentWatcherConfig = segmentWatcherConfig; - - this.isMetadataSegmentCacheEnabled = config.isMetadataSegmentCacheEnable(); - - this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); - this.scheduledExec = Execs.scheduledSingleThreaded("SegmentMetadataView-Cache--%d"); - this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() - .expireAfterAccess(10, TimeUnit.MINUTES) - .build(); - this.segmentMetadataCache = segmentMetadataCache; - } - - @LifecycleStart - public void start() - { - if (!lifecycleLock.canStart()) { - throw new ISE("can't start."); - } - try { - if (isMetadataSegmentCacheEnabled) { - scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); - } - lifecycleLock.started(); - log.info("MetadataSegmentView Started. Configs isMetadataSegmentCacheEnabled [%s]", - isMetadataSegmentCacheEnabled); - } - finally { - lifecycleLock.exitStart(); - } - } - - @LifecycleStop - public void stop() - { - if (!lifecycleLock.canStop()) { - throw new ISE("can't stop."); - } - log.info("MetadataSegmentView is stopping."); - if (isMetadataSegmentCacheEnabled) { - scheduledExec.shutdown(); - } - log.info("MetadataSegmentView Stopped."); - } - - protected Iterator getSegmentTableView() - { - // set of published and realtime segments - final ImmutableSortedSet segments = getSegmentMetadata(); - - // set of available segments and metadata - final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - final List segmentsTableViews = new ArrayList<>(); - - Set seenSegments = new HashSet<>(); - - for (SegmentStatusInCluster segmentStatusInCluster : segments) { - DataSegment segment = segmentStatusInCluster.getDataSegment(); - SegmentId segmentId = segment.getId(); - AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataMap.get(segmentId); - - long numReplicas = 0L, numRows = 0L, isAvailable = 0L; - if (availableSegmentMetadata != null) { - numReplicas = availableSegmentMetadata.getNumReplicas(); - numRows = availableSegmentMetadata.getNumRows(); - isAvailable = 1L; - } - - // Prefer numRows & realtime status info returned from Coordinator. - if (null != segmentStatusInCluster.getNumRows()) - { - numRows = segmentStatusInCluster.getNumRows(); - } - - long isRealtime = Boolean.TRUE.equals(segmentStatusInCluster.isRealtime()) ? 1 : 0; - - // set of segments returned from coordinator include published and realtime segments - // so realtime segments are not published and vice versa - boolean isPublished = !segmentStatusInCluster.isRealtime(); - - // is_active is true for published segments that are not overshadowed - boolean isActive = isPublished && !segmentStatusInCluster.isOvershadowed(); - - SegmentTableView segmentTableView = new SegmentTableView( - segment, - isAvailable, - isRealtime, - numReplicas, - numRows, - // If the segment is unpublished, we won't have this information yet. - // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. - // This should be automatically updated in the next refesh with Coordinator. - segmentStatusInCluster.getReplicationFactor(), - // If the segment is unpublished this value would be false - segmentStatusInCluster.isOvershadowed(), - isPublished, - isActive - ); - - seenSegments.add(segmentId); - segmentsTableViews.add((segmentTableView)); - } - - for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) { - if (seenSegments.contains(availableSegmentMetadataEntry.getKey())) { - continue; - } - AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataEntry.getValue(); - - // since all published segments would have been covered in the previous loop - // the curent set of segments must be unpublished - boolean isPublished = false; - - // is_active is true for unpublished segments iff they are realtime - boolean isActive = availableSegmentMetadata.isRealtime() == 1L; - - SegmentTableView segmentTableView = new SegmentTableView( - availableSegmentMetadata.getSegment(), - // segments announced by historicals or realtime tasks are assumed to be available - 1L, - availableSegmentMetadata.isRealtime(), - availableSegmentMetadata.getNumReplicas(), - availableSegmentMetadata.getNumRows(), - // If the segment is unpublished, we won't have this information yet. - null, - // there is an assumption here that unpublished segments are never overshadowed - false, - isPublished, - isActive - ); - - segmentsTableViews.add(segmentTableView); - } - - log.info("Built the segment table view"); - for (SegmentTableView segmentTableView : segmentsTableViews) { - log.info("SegmentTableView is [%s]", segmentTableView); - } - return segmentsTableViews.iterator(); - } - - private void pollSegmentMetadata() - { - log.info("Polling segment metadata from coordinator"); - - segmentMetadata = fetchSegmentMetadata(); - segmentMetadataCachePopulated.countDown(); - } - - private ImmutableSortedSet getSegmentMetadata() - { - if (isMetadataSegmentCacheEnabled) { - Uninterruptibles.awaitUninterruptibly(segmentMetadataCachePopulated); - return segmentMetadata; - } else { - return fetchSegmentMetadata(); - } - } - - private ImmutableSortedSet fetchSegmentMetadata() - { - final Iterator metadataSegments = - querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); - - final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); - log.info("Polled segments from coordinator"); - while (metadataSegments.hasNext()) { - final SegmentStatusInCluster segment = metadataSegments.next(); - log.info("This is the polled segmentStatusInCluster %s", segment); - final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); - Integer replicationFactor = segment.getReplicationFactor(); - if (replicationFactor == null) { - replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); - } else { - segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); - } - final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( - interned, - segment.isOvershadowed(), - replicationFactor, - segment.getNumRows(), - segment.isRealtime() - ); - log.info("SegmentStatusInCluster %s", segmentStatusInCluster); - builder.add(segmentStatusInCluster); - } - - return builder.build(); - } - - // Note that coordinator must be up to get segments - private JsonParserIterator querySegmentMetadata( - Set watchedDataSources - ) - { - final StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"); - - //queryBuilder.append("&includeRealtimeSegments"); - if (watchedDataSources != null && !watchedDataSources.isEmpty()) { - log.debug( - "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); - for (String ds : watchedDataSources) { - queryBuilder.append("&datasources=").append(ds).append("&"); - } - queryBuilder.setLength(queryBuilder.length() - 1); - } - - String query = queryBuilder.toString(); - log.info("query is %s", query); - return SystemSchema.getThingsFromLeaderNode( - query, - new TypeReference() - { - }, - druidLeaderClient, - objectMapper - ); - } - - private class PollTask implements Runnable - { - @Override - public void run() - { - long delayMS = pollPeriodInMS; - try { - final long pollStartTime = System.nanoTime(); - pollSegmentMetadata(); - final long pollEndTime = System.nanoTime(); - final long pollTimeNS = pollEndTime - pollStartTime; - final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); - delayMS = Math.max(pollPeriodInMS - pollTimeMS, 0); - } - catch (Exception e) { - log.makeAlert(e, "Problem polling Coordinator.").emit(); - } - finally { - if (!Thread.currentThread().isInterrupted()) { - scheduledExec.schedule(new PollTask(), delayMS, TimeUnit.MILLISECONDS); - } - } - } - } -} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index 7f4f626e6ec6..aa1e7965c27e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -46,6 +46,11 @@ public DruidSchema( } } + protected BrokerSegmentMetadataCache cache() + { + return segmentMetadataCache; + } + @Override public Table getTable(String name) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java new file mode 100644 index 000000000000..4f8044c48d45 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.inject.Inject; +import org.apache.druid.client.BrokerSegmentWatcherConfig; +import org.apache.druid.client.DataSegmentInterner; +import org.apache.druid.client.JsonParserIterator; +import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.concurrent.LifecycleLock; +import org.apache.druid.discovery.DruidLeaderClient; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.lifecycle.LifecycleStop; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.SegmentStatusInCluster; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * This class polls the Coordinator in background to keep the latest segments. + * Provides {@link #getSegments()} for others to get segments in metadata store. + * + * The difference between this class and {@link SegmentsMetadataManager} is that this class resides + * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In + * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the + * currently leading Coordinator via HTTP queries. + */ +@ManageLifecycle +public class MetadataSegmentView +{ + private static final EmittingLogger log = new EmittingLogger(MetadataSegmentView.class); + + private final DruidLeaderClient coordinatorDruidLeaderClient; + private final ObjectMapper jsonMapper; + private final BrokerSegmentWatcherConfig segmentWatcherConfig; + + private final boolean isCacheEnabled; + /** + * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and + * sys.segments queries return the segments in sorted order based on segmentId. + * + * Volatile since this reference is reassigned in {@code poll()} and then read in {@code getPublishedSegments()} + * from other threads. + */ + @MonotonicNonNull + private volatile ImmutableSortedSet publishedSegments = null; + /** + * Caches the replication factor for segment IDs. In case of coordinator restarts or leadership re-elections, the coordinator API returns `null` replication factor until load rules are evaluated. + * The cache can be used during these periods to continue serving the previously fetched values. + */ + private final Cache segmentIdToReplicationFactor; + private final ScheduledExecutorService scheduledExec; + private final long pollPeriodInMS; + private final LifecycleLock lifecycleLock = new LifecycleLock(); + private final CountDownLatch cachePopulated = new CountDownLatch(1); + + @Inject + public MetadataSegmentView( + final @Coordinator DruidLeaderClient druidLeaderClient, + final ObjectMapper jsonMapper, + final BrokerSegmentWatcherConfig segmentWatcherConfig, + final BrokerSegmentMetadataCacheConfig config + ) + { + Preconditions.checkNotNull(config, "BrokerSegmentMetadataCacheConfig"); + this.coordinatorDruidLeaderClient = druidLeaderClient; + this.jsonMapper = jsonMapper; + this.segmentWatcherConfig = segmentWatcherConfig; + this.isCacheEnabled = config.isMetadataSegmentCacheEnable(); + this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); + this.scheduledExec = Execs.scheduledSingleThreaded("MetadataSegmentView-Cache--%d"); + this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() + .expireAfterAccess(10, TimeUnit.MINUTES) + .build(); + } + + @LifecycleStart + public void start() + { + if (!lifecycleLock.canStart()) { + throw new ISE("can't start."); + } + try { + if (isCacheEnabled) { + scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); + } + lifecycleLock.started(); + log.info("MetadataSegmentView is started."); + } + finally { + lifecycleLock.exitStart(); + } + } + + @LifecycleStop + public void stop() + { + if (!lifecycleLock.canStop()) { + throw new ISE("can't stop."); + } + log.info("MetadataSegmentView is stopping."); + if (isCacheEnabled) { + scheduledExec.shutdown(); + } + log.info("MetadataSegmentView is stopped."); + } + + private void poll() + { + log.info("Polling segments from coordinator"); + final JsonParserIterator metadataSegments = getMetadataSegments( + coordinatorDruidLeaderClient, + jsonMapper, + segmentWatcherConfig.getWatchedDataSources() + ); + + final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); + while (metadataSegments.hasNext()) { + final SegmentStatusInCluster segment = metadataSegments.next(); + final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); + Integer replicationFactor = segment.getReplicationFactor(); + if (replicationFactor == null) { + replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); + } else { + segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); + } + final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( + interned, + segment.isOvershadowed(), + replicationFactor, + segment.getNumRows(), + segment.isRealtime() + ); + builder.add(segmentStatusInCluster); + } + publishedSegments = builder.build(); + cachePopulated.countDown(); + } + + Iterator getSegments() + { + if (isCacheEnabled) { + Uninterruptibles.awaitUninterruptibly(cachePopulated); + return publishedSegments.iterator(); + } else { + return getMetadataSegments( + coordinatorDruidLeaderClient, + jsonMapper, + segmentWatcherConfig.getWatchedDataSources() + ); + } + } + + // Note that coordinator must be up to get segments + private JsonParserIterator getMetadataSegments( + DruidLeaderClient coordinatorClient, + ObjectMapper jsonMapper, + Set watchedDataSources + ) + { + StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments"); + if (watchedDataSources != null && !watchedDataSources.isEmpty()) { + log.debug( + "Filtering datasources in segments based on broker's watchedDataSources[%s]", watchedDataSources); + final StringBuilder sb = new StringBuilder(); + for (String ds : watchedDataSources) { + sb.append("datasources=").append(ds).append("&"); + } + sb.setLength(sb.length() - 1); + queryBuilder.append("&"); + queryBuilder.append(sb); + } + + return SystemSchema.getThingsFromLeaderNode( + queryBuilder.toString(), + new TypeReference() + { + }, + coordinatorClient, + jsonMapper + ); + } + + private class PollTask implements Runnable + { + @Override + public void run() + { + long delayMS = pollPeriodInMS; + try { + final long pollStartTime = System.nanoTime(); + poll(); + final long pollEndTime = System.nanoTime(); + final long pollTimeNS = pollEndTime - pollStartTime; + final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); + delayMS = Math.max(pollPeriodInMS - pollTimeMS, 0); + } + catch (Exception e) { + log.makeAlert(e, "Problem polling Coordinator.").emit(); + } + finally { + if (!Thread.currentThread().isInterrupted()) { + scheduledExec.schedule(new PollTask(), delayMS, TimeUnit.MILLISECONDS); + } + } + } + } + +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index afa0986467d5..48e3a4a44268 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -27,6 +27,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; import com.google.common.util.concurrent.Futures; import com.google.inject.Inject; @@ -63,10 +66,10 @@ import org.apache.druid.java.util.http.client.Request; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; -import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.server.DruidNode; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; @@ -80,6 +83,7 @@ import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.SegmentStatusInCluster; import org.jboss.netty.handler.codec.http.HttpMethod; import javax.annotation.Nullable; @@ -94,6 +98,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; public class SystemSchema extends AbstractSchema @@ -104,10 +109,10 @@ public class SystemSchema extends AbstractSchema private static final String TASKS_TABLE = "tasks"; private static final String SUPERVISOR_TABLE = "supervisors"; - private static final Function> - SEGMENT_STATUS_IN_CLUSTER_RA_GENERATOR = segmentTableView -> + private static final Function> + SEGMENT_STATUS_IN_CLUSTER_RA_GENERATOR = segment -> Collections.singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply( - segmentTableView.getSegment().getDataSource()) + segment.getDataSegment().getDataSource()) ); private static final Function> SEGMENT_RA_GENERATOR = @@ -115,6 +120,21 @@ public class SystemSchema extends AbstractSchema segment.getDataSource()) ); + private static final long REPLICATION_FACTOR_UNKNOWN = -1L; + + /** + * Booleans constants represented as long type, + * where 1 = true and 0 = false to make it easy to count number of segments + * which are published, available etc. + */ + private static final long IS_ACTIVE_FALSE = 0L; + private static final long IS_ACTIVE_TRUE = 1L; + private static final long IS_PUBLISHED_FALSE = 0L; + private static final long IS_PUBLISHED_TRUE = 1L; + private static final long IS_AVAILABLE_TRUE = 1L; + private static final long IS_OVERSHADOWED_FALSE = 0L; + private static final long IS_OVERSHADOWED_TRUE = 1L; + static final RowSignature SEGMENTS_SIGNATURE = RowSignature .builder() .add("segment_id", ColumnType.STRING) @@ -192,7 +212,8 @@ public class SystemSchema extends AbstractSchema @Inject public SystemSchema( - final BrokerSegmentMetadataView metadataView, + final DruidSchema druidSchema, + final MetadataSegmentView metadataView, final TimelineServerView serverView, final FilteredServerInventoryView serverInventoryView, final AuthorizerMapper authorizerMapper, @@ -204,7 +225,7 @@ public SystemSchema( { Preconditions.checkNotNull(serverView, "serverView"); this.tableMap = ImmutableMap.of( - SEGMENTS_TABLE, new SegmentsTable(metadataView, jsonMapper, authorizerMapper), + SEGMENTS_TABLE, new SegmentsTable(druidSchema, metadataView, jsonMapper, authorizerMapper), SERVERS_TABLE, new ServersTable(druidNodeDiscoveryProvider, serverInventoryView, authorizerMapper, overlordClient, coordinatorDruidLeaderClient), SERVER_SEGMENTS_TABLE, new ServerSegmentsTable(serverView, authorizerMapper), TASKS_TABLE, new TasksTable(overlordClient, authorizerMapper), @@ -223,16 +244,19 @@ public Map getTableMap() */ static class SegmentsTable extends AbstractTable implements ScannableTable { + private final DruidSchema druidSchema; private final ObjectMapper jsonMapper; private final AuthorizerMapper authorizerMapper; - private final BrokerSegmentMetadataView metadataView; + private final MetadataSegmentView metadataView; public SegmentsTable( - BrokerSegmentMetadataView metadataView, + DruidSchema druidSchemna, + MetadataSegmentView metadataView, ObjectMapper jsonMapper, AuthorizerMapper authorizerMapper ) { + this.druidSchema = druidSchemna; this.metadataView = metadataView; this.jsonMapper = jsonMapper; this.authorizerMapper = authorizerMapper; @@ -253,12 +277,50 @@ public TableType getJdbcTableType() @Override public Enumerable scan(DataContext root) { - final Iterator segmentTableView = metadataView.getSegmentTableView(); + //get available segments from druidSchema + final Map availableSegmentMetadata = + druidSchema.cache().getSegmentMetadataSnapshot(); + final Iterator> availableSegmentEntries = + availableSegmentMetadata.entrySet().iterator(); + + // in memory map to store segment data from available segments + final Map partialSegmentDataMap = + Maps.newHashMapWithExpectedSize(druidSchema.cache().getTotalSegments()); + for (AvailableSegmentMetadata h : availableSegmentMetadata.values()) { + PartialSegmentData partialSegmentData = + new PartialSegmentData(IS_AVAILABLE_TRUE, h.isRealtime(), h.getNumReplicas(), h.getNumRows()); + partialSegmentDataMap.put(h.getSegment().getId(), partialSegmentData); + } + + // Get segments from metadata segment cache (if enabled in SQL planner config), else directly from + // Coordinator. + // this may include both published and realtime segments. + final Iterator metadataStoreSegments = metadataView.getSegments(); + + final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(druidSchema.cache().getTotalSegments()); - final FluentIterable allSegments = FluentIterable - .from(() -> getAuthorizedPublishedSegments(segmentTableView, root)) + final FluentIterable publishedSegments = FluentIterable + .from(() -> getAuthorizedPublishedSegments(metadataStoreSegments, root)) .transform(val -> { - final DataSegment segment = val.getSegment(); + final DataSegment segment = val.getDataSegment(); + segmentsAlreadySeen.add(segment.getId()); + final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(segment.getId()); + long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; + if (partialSegmentData != null) { + numReplicas = partialSegmentData.getNumReplicas(); + numRows = partialSegmentData.getNumRows(); + isAvailable = partialSegmentData.isAvailable(); + } + + isRealtime = Boolean.TRUE.equals(val.isRealtime()) ? 1 : 0; + + // set of segments returned from coordinator include published and realtime segments + // so realtime segments are not published and vice versa + boolean isPublished = !val.isRealtime(); + + // is_active is true for published segments that are not overshadowed + boolean isActive = isPublished && !val.isOvershadowed(); + try { return new Object[]{ segment.getId(), @@ -268,20 +330,64 @@ public Enumerable scan(DataContext root) segment.getSize(), segment.getVersion(), (long) segment.getShardSpec().getPartitionNum(), - val.getNumReplicas(), - val.getNumRows(), - val.isActive(), - val.isPublished(), - val.isAvailable(), - val.isRealtime(), - val.isOvershadowed(), + numReplicas, + numRows, + isActive ? IS_ACTIVE_TRUE : IS_ACTIVE_FALSE, + isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, + isAvailable, + isRealtime, + val.isOvershadowed() ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE, segment.getShardSpec() == null ? null : jsonMapper.writeValueAsString(segment.getShardSpec()), segment.getDimensions() == null ? null : jsonMapper.writeValueAsString(segment.getDimensions()), segment.getMetrics() == null ? null : jsonMapper.writeValueAsString(segment.getMetrics()), - segment.getLastCompactionState() == null - ? null - : jsonMapper.writeValueAsString(segment.getLastCompactionState()), - val.getReplicationFactor() + segment.getLastCompactionState() == null ? null : jsonMapper.writeValueAsString(segment.getLastCompactionState()), + // If the segment is unpublished, we won't have this information yet. + // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. + // This should be automatically updated in the next refesh with Coordinator. + val.getReplicationFactor() == null ? REPLICATION_FACTOR_UNKNOWN : (long) val.getReplicationFactor() + }; + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + + final FluentIterable availableSegments = FluentIterable + .from(() -> getAuthorizedAvailableSegments( + availableSegmentEntries, + root + )) + .transform(val -> { + if (segmentsAlreadySeen.contains(val.getKey())) { + return null; + } + final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(val.getKey()); + final long numReplicas = partialSegmentData == null ? 0L : partialSegmentData.getNumReplicas(); + try { + return new Object[]{ + val.getKey(), + val.getKey().getDataSource(), + val.getKey().getInterval().getStart().toString(), + val.getKey().getInterval().getEnd().toString(), + val.getValue().getSegment().getSize(), + val.getKey().getVersion(), + (long) val.getValue().getSegment().getShardSpec().getPartitionNum(), + numReplicas, + val.getValue().getNumRows(), + // is_active is true for unpublished segments iff they are realtime + val.getValue().isRealtime() /* is_active */, + // is_published is false for unpublished segments + IS_PUBLISHED_FALSE, + // is_available is assumed to be always true for segments announced by historicals or realtime tasks + IS_AVAILABLE_TRUE, + val.getValue().isRealtime(), + IS_OVERSHADOWED_FALSE, + // there is an assumption here that unpublished segments are never overshadowed + val.getValue().getSegment().getShardSpec() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getShardSpec()), + val.getValue().getSegment().getDimensions() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getDimensions()), + val.getValue().getSegment().getMetrics() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getMetrics()), + null, // unpublished segments from realtime tasks will not be compacted yet + REPLICATION_FACTOR_UNKNOWN // If the segment is unpublished, we won't have this information yet. }; } catch (JsonProcessingException e) { @@ -289,12 +395,16 @@ public Enumerable scan(DataContext root) } }); + final Iterable allSegments = Iterables.unmodifiableIterable( + Iterables.concat(publishedSegments, availableSegments) + ); return Linq4j.asEnumerable(allSegments).where(Objects::nonNull); + } - private Iterator getAuthorizedPublishedSegments( - Iterator it, + private Iterator getAuthorizedPublishedSegments( + Iterator it, DataContext root ) { @@ -303,7 +413,7 @@ private Iterator getAuthorizedPublishedSegments( "authenticationResult in dataContext" ); - final Iterable authorizedSegments = AuthorizationUtils + final Iterable authorizedSegments = AuthorizationUtils .filterAuthorizedResources( authenticationResult, () -> it, @@ -339,67 +449,35 @@ private Iterator> getAuthorizedAvaila return authorizedSegments.iterator(); } - protected static class SegmentTableView + private static class PartialSegmentData { - /** - * Booleans constants represented as long type, - * where 1 = true and 0 = false to make it easy to count number of segments - * which are published, available etc. - */ - private static final long IS_ACTIVE_FALSE = 0L; - private static final long IS_ACTIVE_TRUE = 1L; - private static final long IS_PUBLISHED_FALSE = 0L; - private static final long IS_PUBLISHED_TRUE = 1L; - private static final long IS_OVERSHADOWED_FALSE = 0L; - private static final long IS_OVERSHADOWED_TRUE = 1L; - private static final long REPLICATION_FACTOR_UNKNOWN = -1L; - - private final DataSegment segment; - private final long available; - private final long realtime; + private final long isAvailable; + private final long isRealtime; private final long numReplicas; private final long numRows; - private final long replicationFactor; - private final long overshadowed; - private final long published; - private final long active; - - public SegmentTableView( - DataSegment segment, - long isAvailable, - long realtime, - long numReplicas, - long numRows, - Integer replicationFactor, - boolean overshadowed, - boolean isPublished, - boolean active + + public PartialSegmentData( + final long isAvailable, + final long isRealtime, + final long numReplicas, + final long numRows ) + { - this.segment = segment; - this.available = isAvailable; - this.realtime = realtime; + this.isAvailable = isAvailable; + this.isRealtime = isRealtime; this.numReplicas = numReplicas; this.numRows = numRows; - this.replicationFactor = (null == replicationFactor) ? REPLICATION_FACTOR_UNKNOWN : (long) replicationFactor; - this.overshadowed = overshadowed ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE; - this.published = isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE; - this.active = active ? IS_ACTIVE_TRUE : IS_ACTIVE_FALSE; - } - - public DataSegment getSegment() - { - return segment; } public long isAvailable() { - return available; + return isAvailable; } public long isRealtime() { - return realtime; + return isRealtime; } public long getNumReplicas() @@ -411,41 +489,6 @@ public long getNumRows() { return numRows; } - - public long getReplicationFactor() - { - return replicationFactor; - } - - public long isOvershadowed() - { - return overshadowed; - } - - public long isPublished() - { - return published; - } - - public long isActive() - { - return active; - } - - @Override - public String toString() - { - return "SegmentTableView{" + - "segmentId=" + segment.getId() + - ", isAvailable=" + available + - ", isRealtime=" + realtime + - ", numReplicas=" + numReplicas + - ", numRows=" + numRows + - ", replicationFactor=" + replicationFactor + - ", isOvershadowed=" + overshadowed + - ", isPublished=" + published + - '}'; - } } } diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 63ef71585dd9..56d0d2d5d41f 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -62,6 +62,7 @@ public class SqlModule implements Module public static final String PROPERTY_SQL_VIEW_MANAGER_TYPE = "druid.sql.viewmanager.type"; public static final String PROPERTY_SQL_SCHEMA_MANAGER_TYPE = "druid.sql.schemamanager.type"; public static final String PROPERTY_SQL_APPROX_COUNT_DISTINCT_CHOICE = "druid.sql.approxCountDistinct.function"; + private Properties props; @Inject diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index a159dcc24e5c..ed7f083dda89 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -19,7 +19,6 @@ package org.apache.druid.sql.calcite.schema; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; @@ -37,12 +36,11 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.common.config.NullHandling; @@ -64,7 +62,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.http.client.Request; -import org.apache.druid.java.util.http.client.response.FullResponseHolder; import org.apache.druid.java.util.http.client.response.HttpResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; @@ -82,13 +79,16 @@ import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.DruidNode; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.coordinator.BytesAccumulatingResponseHandler; +import org.apache.druid.server.coordinator.simulate.TestServerInventoryView; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; @@ -97,26 +97,18 @@ import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.server.security.ResourceType; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable; -import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; -import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; -import org.jboss.netty.handler.codec.http.DefaultHttpResponse; -import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponse; -import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.jboss.netty.handler.codec.http.HttpVersion; import org.joda.time.DateTime; import org.junit.AfterClass; import org.junit.Assert; @@ -130,7 +122,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -162,6 +153,7 @@ public class SystemSchemaTest extends CalciteTestBase ); private SystemSchema schema; + private SpecificSegmentsQuerySegmentWalker walker; private DruidLeaderClient client; private DruidLeaderClient coordinatorClient; private OverlordClient overlordClient; @@ -170,10 +162,11 @@ public class SystemSchemaTest extends CalciteTestBase private StringFullResponseHolder responseHolder; private BytesAccumulatingResponseHandler responseHandler; private Request request; + private DruidSchema druidSchema; private AuthorizerMapper authMapper; private static QueryRunnerFactoryConglomerate conglomerate; private static Closer resourceCloser; - private BrokerSegmentMetadataView metadataView; + private MetadataSegmentView metadataView; private DruidNodeDiscoveryProvider druidNodeDiscoveryProvider; private FilteredServerInventoryView serverInventoryView; @@ -254,7 +247,7 @@ public void setUp() throws Exception .rows(ROWS3) .buildMMappedIndex(); - SpecificSegmentsQuerySegmentWalker walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) + walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) .add(segment1, index1) .add(segment2, index2) .add(segment3, index3); @@ -272,72 +265,14 @@ public void setUp() throws Exception ), new NoopCoordinatorClient() ); - cache.start(); cache.awaitInitialization(); - - DruidNode coordinatorNode = new DruidNode("test-coordinator", "localhost", false, 8081, null, true, false); - DruidLeaderClient druidLeaderClient = new DruidLeaderClient( - new CalciteTests.FakeHttpClient(), - new CalciteTests.FakeDruidNodeDiscoveryProvider( - ImmutableMap.of( - NodeRole.COORDINATOR, - new CalciteTests.FakeDruidNodeDiscovery(ImmutableMap.of(NodeRole.COORDINATOR, coordinatorNode)) - ) - ), - NodeRole.COORDINATOR, - "/simple/leader" - ) - { - @Override - public String findCurrentLeader() - { - return coordinatorNode.getHostAndPortToUse(); - } - - @Override - public Request makeRequest(HttpMethod httpMethod, String urlPath) throws IOException - { - return new Request(httpMethod, new URL(StringUtils.format("http://%s%s", coordinatorNode.getHostAndPortToUse(), urlPath))); - } - - @Override - public > H go(Request request, HttpResponseHandler responseHandler) - throws IOException - { - List segmentStatusInClusterList = new ArrayList<>(Arrays.asList( - new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 0L, false), - new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 0L, false), - new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 0L, false), - new SegmentStatusInCluster(segment1, true, 2, 3L, false), - new SegmentStatusInCluster(segment2, false, 0, 3L, false) - )); - - InputStreamFullResponseHolder responseHolder = new InputStreamFullResponseHolder(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK)); - byte[] bytesToWrite = mapper.writeValueAsString(segmentStatusInClusterList).getBytes(StandardCharsets.UTF_8); - responseHolder.addChunk(bytesToWrite); - responseHolder.done(); - return (H) responseHolder; - } - }; - - metadataView = new BrokerSegmentMetadataView( - druidLeaderClient, - mapper, - new BrokerSegmentWatcherConfig(), - new BrokerSegmentMetadataCacheConfig() { - @Override - public boolean isMetadataSegmentCacheEnable() - { - return false; - } - }, - cache - ); - + druidSchema = new DruidSchema(cache, null); + metadataView = EasyMock.createMock(MetadataSegmentView.class); druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class); schema = new SystemSchema( + druidSchema, metadataView, serverView, serverInventoryView, @@ -349,17 +284,6 @@ public boolean isMetadataSegmentCacheEnable() ); } - @Test - public void test1() throws IOException - { - SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster(segment1, true, 2, 4L, Boolean.TRUE); - byte[] x = mapper.writeValueAsBytes(segmentStatusInCluster); - SegmentStatusInCluster w = mapper.readValue(x, new TypeReference() - { - }); - int p = 1; - } - private final CompactionState expectedCompactionState = new CompactionState( new DynamicPartitionsSpec(null, null), null, @@ -645,9 +569,18 @@ public void testGetTableMap() @Test public void testSegmentsTable() throws Exception { - final SegmentsTable segmentsTable = new SegmentsTable(metadataView, new ObjectMapper(), authMapper); + final SegmentsTable segmentsTable = new SegmentsTable(druidSchema, metadataView, new ObjectMapper(), authMapper); + final Set publishedSegments = new HashSet<>(Arrays.asList( + new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, null, false), + new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, null, false), + new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, null, false), + new SegmentStatusInCluster(segment1, true, 2, null, false), + new SegmentStatusInCluster(segment2, false, 0, null, false) + )); + + EasyMock.expect(metadataView.getSegments()).andReturn(publishedSegments.iterator()).once(); - EasyMock.replay(client, request, responseHolder, responseHandler); + EasyMock.replay(client, request, responseHolder, responseHandler, metadataView); DataContext dataContext = createDataContext(Users.SUPER); final List rows = segmentsTable.scan(dataContext).toList(); rows.sort((Object[] row1, Object[] row2) -> ((Comparable) row1[0]).compareTo(row2[0])); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index 66457b292f83..fc6ad94fe1aa 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -31,11 +30,8 @@ import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ServerInventoryView; import org.apache.druid.client.ServerView; -import org.apache.druid.client.TimelineServerView; -import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.client.indexing.NoopOverlordClient; import org.apache.druid.discovery.DiscoveryDruidNode; import org.apache.druid.discovery.DruidLeaderClient; @@ -53,16 +49,12 @@ import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.join.JoinableFactoryWrapper; -import org.apache.druid.segment.join.MapJoinableFactory; -import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.metadata.TestTimelineServerView; -import org.apache.druid.server.SegmentManager; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.DruidNode; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryScheduler; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; @@ -80,15 +72,12 @@ import org.apache.druid.sql.calcite.planner.PlannerFactory; import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.SqlEngine; -import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; -import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataBuilder; +import org.apache.druid.sql.calcite.schema.MetadataSegmentView; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.timeline.DataSegment; -import org.easymock.EasyMock; import org.joda.time.Duration; import javax.annotation.Nullable; @@ -343,7 +332,7 @@ public static DruidOperatorTable createOperatorTable() } public static SystemSchema createMockSystemSchema( - final QueryLifecycleFactory queryLifecycleFactory, + final DruidSchema druidSchema, final SpecificSegmentsQuerySegmentWalker walker, final AuthorizerMapper authorizerMapper ) @@ -383,29 +372,15 @@ public ListenableFuture findCurrentLeader() } }; - TimelineServerView timelineServerView = new TestTimelineServerView(walker.getSegments()); - return new SystemSchema( - new BrokerSegmentMetadataView( + druidSchema, + new MetadataSegmentView( druidLeaderClient, getJsonMapper(), new BrokerSegmentWatcherConfig(), - BrokerSegmentMetadataCacheConfig.create(), - new BrokerSegmentMetadataCache( - queryLifecycleFactory, - timelineServerView, - BrokerSegmentMetadataCacheConfig.create(), - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder( - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - new SegmentManager(EasyMock.createMock(SegmentLoader.class)) - ), - new NoopCoordinatorClient() - ) + BrokerSegmentMetadataCacheConfig.create() ), - timelineServerView, + new TestTimelineServerView(walker.getSegments()), new FakeServerInventoryView(), authorizerMapper, druidLeaderClient, @@ -433,7 +408,7 @@ public static DruidSchemaCatalog createMockRootSchema( /** * A fake {@link HttpClient} for {@link #createMockSystemSchema}. */ - public static class FakeHttpClient implements HttpClient + private static class FakeHttpClient implements HttpClient { @Override public ListenableFuture go( @@ -458,7 +433,7 @@ public ListenableFuture go( /** * A fake {@link DruidNodeDiscoveryProvider} for {@link #createMockSystemSchema}. */ - public static class FakeDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider + private static class FakeDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider { private final Map nodeDiscoveries; @@ -484,7 +459,7 @@ public DruidNodeDiscovery getForNodeRole(NodeRole nodeRole) } } - public static class FakeDruidNodeDiscovery implements DruidNodeDiscovery + private static class FakeDruidNodeDiscovery implements DruidNodeDiscovery { private final Set nodes; @@ -493,7 +468,7 @@ public static class FakeDruidNodeDiscovery implements DruidNodeDiscovery this.nodes = new HashSet<>(); } - public FakeDruidNodeDiscovery(Map nodes) + FakeDruidNodeDiscovery(Map nodes) { this.nodes = Sets.newHashSetWithExpectedSize(nodes.size()); nodes.forEach((k, v) -> { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 9f540d634d06..dc716b2a45c1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -150,7 +150,7 @@ public static DruidSchemaCatalog createMockRootSchema( druidSchemaManager ); SystemSchema systemSchema = - CalciteTests.createMockSystemSchema(createMockQueryLifecycleFactory(walker, conglomerate), walker, authorizerMapper); + CalciteTests.createMockSystemSchema(druidSchema, walker, authorizerMapper); LookupSchema lookupSchema = createMockLookupSchema(injector); ViewSchema viewSchema = viewManager != null ? new ViewSchema(viewManager) : null; From 151b0b1f00707d8562da956e2715c986613f6709 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 9 Sep 2023 18:25:58 +0530 Subject: [PATCH 13/82] undo changes in SegmentsMetadataManager --- .../org/apache/druid/metadata/SegmentsMetadataManager.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java index eecf8e036195..a774afcd47b3 100644 --- a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java +++ b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java @@ -33,6 +33,11 @@ import java.util.List; import java.util.Set; +/** + * The difference between this class and org.apache.druid.sql.calcite.schema.MetadataSegmentView is that this class + * resides in Coordinator's memory, while org.apache.druid.sql.calcite.schema.MetadataSegmentView resides in Broker's + * memory. + */ public interface SegmentsMetadataManager { void startPollingDatabasePeriodically(); From 8dbea5b9472c47fce42ee54c325ac648a263b00a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 9 Sep 2023 21:51:17 +0530 Subject: [PATCH 14/82] Minor code changes and add multiple tests --- .../metadata/SegmentMetadataCache.java | 18 +-- .../druid/server/http/MetadataResource.java | 2 +- .../CoordinatorClientImplTest.java | 30 +++++ .../metadata/SegmentMetadataCacheCommon.java | 13 +- .../server/http/MetadataResourceTest.java | 58 +++++++- .../schema/BrokerSegmentMetadataCache.java | 20 ++- .../BrokerSegmentMetadataCacheConfigTest.java | 94 +++++++++++++ ...PhysicalDataSourceMetadataBuilderTest.java | 126 ++++++++++++++++++ 8 files changed, 321 insertions(+), 40 deletions(-) create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 32feb8844179..704fc52de5f8 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -438,7 +438,7 @@ public void rebuildDatasource(String dataSource) if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { - log.info("[%s] signature is unchanged.", dataSource); + log.debug("[%s] signature is unchanged.", dataSource); } } @@ -459,7 +459,7 @@ public DataSourceInformation getDatasource(String name) return tables.get(name); } - public Map getDataSourceSchemaMap() + public Map getDataSourceInformationMap() { return ImmutableMap.copyOf(tables); } @@ -877,7 +877,7 @@ public int getTotalSegments() } @VisibleForTesting - public TreeSet getSegmentsNeedingRefresh() + TreeSet getSegmentsNeedingRefresh() { synchronized (lock) { return segmentsNeedingRefresh; @@ -885,7 +885,7 @@ public TreeSet getSegmentsNeedingRefresh() } @VisibleForTesting - public TreeSet getMutableSegments() + TreeSet getMutableSegments() { synchronized (lock) { return mutableSegments; @@ -893,19 +893,13 @@ public TreeSet getMutableSegments() } @VisibleForTesting - public Set getDataSourcesNeedingRebuild() + Set getDataSourcesNeedingRebuild() { synchronized (lock) { return dataSourcesNeedingRebuild; } } - Object getLock() - { - return lock; - } - - /** * Execute a SegmentMetadata query and return a {@link Sequence} of {@link SegmentAnalysis}. * @@ -951,7 +945,7 @@ protected Sequence runSegmentMetadataQuery( } @VisibleForTesting - static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) + public static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) { final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); for (Map.Entry entry : analysis.getColumns().entrySet()) { diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 1e86954ea2fe..447fac8784aa 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -369,7 +369,7 @@ public Response getDataSourceInformation( if (null == segmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } - Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceSchemaMap(); + Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceInformationMap(); List results = new ArrayList<>(); List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; diff --git a/server/src/test/java/org/apache/druid/client/coordinator/CoordinatorClientImplTest.java b/server/src/test/java/org/apache/druid/client/coordinator/CoordinatorClientImplTest.java index f48e21327a0b..e418c2772318 100644 --- a/server/src/test/java/org/apache/druid/client/coordinator/CoordinatorClientImplTest.java +++ b/server/src/test/java/org/apache/druid/client/coordinator/CoordinatorClientImplTest.java @@ -28,6 +28,9 @@ import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.MockServiceClient; import org.apache.druid.rpc.RequestBuilder; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.jboss.netty.handler.codec.http.HttpMethod; @@ -168,4 +171,31 @@ public void test_fetchUsedSegments() throws Exception coordinatorClient.fetchUsedSegments("xyz", intervals).get() ); } + + @Test + public void test_fetchDataSourceInformation() throws Exception + { + String foo = "foo"; + + DataSourceInformation fooInfo = new DataSourceInformation( + "foo", + RowSignature.builder() + .add("d1", ColumnType.FLOAT) + .add("d2", ColumnType.DOUBLE) + .build() + ); + + serviceClient.expectAndRespond( + new RequestBuilder(HttpMethod.POST, "/druid/coordinator/v1/metadata/dataSourceInformation") + .jsonContent(jsonMapper, Collections.singletonList(foo)), + HttpResponseStatus.OK, + ImmutableMap.of(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON), + jsonMapper.writeValueAsBytes(Collections.singletonList(fooInfo)) + ); + + Assert.assertEquals( + Collections.singletonList(fooInfo), + coordinatorClient.fetchDataSourceInformation(Collections.singleton(foo)).get() + ); + } } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index a391ba9fb74c..0bd83f88f2d2 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -86,8 +86,10 @@ public abstract class SegmentMetadataCacheCommon static QueryRunnerFactoryConglomerate conglomerate; static Closer resourceCloser; - static QueryToolChestWarehouse queryToolChestWarehouse; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @BeforeClass public static void setUpClass() @@ -110,15 +112,6 @@ public static void tearDownClass() throws IOException resourceCloser.close(); } - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - - @Before - public void setUpCommon() - { - } - InputRow createRow(final ImmutableMap map) { return MapInputRowParser.parse(FOO_SCHEMA, (Map) map); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index f24cc7bd2c76..141c0ddc5c39 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -24,13 +24,15 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -47,10 +49,10 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; public class MetadataResourceTest { @@ -126,7 +128,7 @@ public void testGetAllSegmentsWithOvershadowedStatus() { Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", null); - final List resultList = extractSegmentStatusList(response); + final List resultList = extractResponseList(response); Assert.assertEquals(resultList.size(), 4); Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, null, false), resultList.get(0)); Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, null, false), resultList.get(1)); @@ -223,7 +225,7 @@ public void testGetAllSegmentsIncludingRealtime() Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", "includeRealtimeSegments"); - final List resultList = extractSegmentStatusList(response); + final List resultList = extractResponseList(response); Assert.assertEquals(resultList.size(), 6); Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 20L, false), resultList.get(0)); Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 30L, false), resultList.get(1)); @@ -234,6 +236,50 @@ public void testGetAllSegmentsIncludingRealtime() Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[1], false, null, 40L, true), resultList.get(5)); } + @Test + public void testGetDataSourceInformation() { + SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); + Map dataSourceInformationMap = new HashMap<>(); + + dataSourceInformationMap.put( + DATASOURCE1, + new DataSourceInformation( + DATASOURCE1, + RowSignature.builder() + .add("c1", ColumnType.FLOAT) + .add("c2", ColumnType.DOUBLE) + .build() + ) + ); + + dataSourceInformationMap.put( + "datasource2", + new DataSourceInformation( + "datasource2", + RowSignature.builder() + .add("d1", ColumnType.FLOAT) + .add("d2", ColumnType.DOUBLE) + .build() + ) + ); + + Mockito.doReturn(dataSourceInformationMap).when(segmentMetadataCache).getDataSourceInformationMap(); + + metadataResource = new MetadataResource( + segmentsMetadataManager, + storageCoordinator, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + coordinator, + segmentMetadataCache + ); + + Response response = metadataResource.getDataSourceInformation(Collections.singletonList(DATASOURCE1)); + + List dataSourceInformations = extractResponseList(response); + Assert.assertEquals(dataSourceInformations.size(), 1); + Assert.assertEquals(dataSourceInformations.get(0), dataSourceInformationMap.get(DATASOURCE1)); + } + @Test public void testGetSegment() { @@ -259,10 +305,10 @@ public void testGetSegment() ); } - private List extractSegmentStatusList(Response response) + private List extractResponseList(Response response) { return Lists.newArrayList( - (Iterable) response.getEntity() + (Iterable) response.getEntity() ); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 0a0428ad1d50..8a037e7cd039 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -87,43 +87,41 @@ public void refresh(final Set segmentsToRefresh, final Set da segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); - Map polledDataSourceSchema = new HashMap<>(); + Map polledDataSourceMetadata = new HashMap<>(); - // Fetch dataSource schema from the Coordinator + // Fetch dataSource information from the Coordinator try { FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) - .forEach(item -> polledDataSourceSchema.put( + .forEach(item -> polledDataSourceMetadata.put( item.getDatasource(), physicalDatasourceMetadataBuilder.build(item) )); } catch (Exception e) { - log.error("Exception querying coordinator for schema"); + log.warn(e, "Exception querying coordinator to fetch dataSourceInformation."); } - log.info("Queried ds schema are [%s]", polledDataSourceSchema); - - tables.putAll(polledDataSourceSchema); + tables.putAll(polledDataSourceMetadata); // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.forEach(segment -> { - if (polledDataSourceSchema.containsKey(segment.getDataSource())) { + if (polledDataSourceMetadata.containsKey(segment.getDataSource())) { segmentsToRefresh.remove(segment); } }); - // Refresh the segments. + // Refresh the remaining segments. final Set refreshed = refreshSegments(segmentsToRefresh); synchronized (lock) { // Add missing segments back to the refresh list. - getSegmentsNeedingRefresh().addAll(Sets.difference(segmentsToRefresh, refreshed)); + segmentsNeedingRefresh.addAll(Sets.difference(segmentsToRefresh, refreshed)); // Compute the list of dataSources to rebuild tables for. dataSourcesToRebuild.addAll(dataSourcesNeedingRebuild); refreshed.forEach(segment -> dataSourcesToRebuild.add(segment.getDataSource())); // Remove those dataSource for which we received schema from the Coordinator. - dataSourcesToRebuild.removeAll(polledDataSourceSchema.keySet()); + dataSourcesToRebuild.removeAll(polledDataSourceMetadata.keySet()); dataSourcesNeedingRebuild.clear(); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java new file mode 100644 index 000000000000..25e6096bb190 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Injector; +import org.apache.druid.guice.GuiceInjectors; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.JsonConfigurator; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.sql.calcite.planner.CalcitePlannerModule; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +public class BrokerSegmentMetadataCacheConfigTest +{ + + private static final String CONFIG_BASE = CalcitePlannerModule.CONFIG_BASE; + + @Test + public void testDefaultConfig() + { + final Injector injector = createInjector(); + final JsonConfigProvider provider = JsonConfigProvider.of( + CONFIG_BASE, + BrokerSegmentMetadataCacheConfig.class + ); + + final Properties properties = new Properties(); + provider.inject(properties, injector.getInstance(JsonConfigurator.class)); + final BrokerSegmentMetadataCacheConfig config = provider.get(); + Assert.assertTrue(config.isAwaitInitializationOnStart()); + Assert.assertTrue(config.isMetadataSegmentCacheEnable()); + Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); + Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); + } + + @Test + public void testCustomizedConfig() + { + final Injector injector = createInjector(); + final JsonConfigProvider provider = JsonConfigProvider.of( + CONFIG_BASE, + BrokerSegmentMetadataCacheConfig.class + ); + final Properties properties = new Properties(); + properties.setProperty( + CONFIG_BASE + ".metadataColumnTypeMergePolicy", + "latestInterval" + ); + properties.setProperty(CONFIG_BASE + ".metadataRefreshPeriod", "PT2M"); + properties.setProperty(CONFIG_BASE + ".awaitInitializationOnStart", "false"); + provider.inject(properties, injector.getInstance(JsonConfigurator.class)); + final BrokerSegmentMetadataCacheConfig config = provider.get(); + Assert.assertFalse(config.isAwaitInitializationOnStart()); + Assert.assertTrue(config.isMetadataSegmentCacheEnable()); + Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); + Assert.assertEquals( + new SegmentMetadataCache.FirstTypeMergePolicy(), + config.getMetadataColumnTypeMergePolicy() + ); + } + + private Injector createInjector() + { + return GuiceInjectors.makeStartupInjectorWithModules( + ImmutableList.of( + binder -> { + JsonConfigProvider.bind(binder, CONFIG_BASE, BrokerSegmentMetadataCacheConfig.class); + } + ) + ); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java new file mode 100644 index 000000000000..dc670af203e4 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.google.common.collect.Sets; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.join.JoinConditionAnalysis; +import org.apache.druid.segment.join.Joinable; +import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.DataSourceInformation; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; + +public class PhysicalDataSourceMetadataBuilderTest +{ + private CountDownLatch getDatasourcesLatch = new CountDownLatch(1); + + private Set segmentDataSourceNames; + private Set joinableDataSourceNames; + private SegmentManager segmentManager; + private JoinableFactory globalTableJoinable; + + private PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + + @Before + public void setUp() throws Exception + { + segmentDataSourceNames = Sets.newConcurrentHashSet(); + joinableDataSourceNames = Sets.newConcurrentHashSet(); + segmentManager = new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + { + @Override + public Set getDataSourceNames() + { + getDatasourcesLatch.countDown(); + return segmentDataSourceNames; + } + }; + + globalTableJoinable = new JoinableFactory() + { + @Override + public boolean isDirectlyJoinable(DataSource dataSource) + { + return dataSource instanceof GlobalTableDataSource && + joinableDataSourceNames.contains(((GlobalTableDataSource) dataSource).getName()); + } + + @Override + public Optional build( + DataSource dataSource, + JoinConditionAnalysis condition + ) + { + return Optional.empty(); + } + }; + + physicalDatasourceMetadataBuilder = new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager); + } + + @Test + public void testBuild() { + segmentDataSourceNames.add("foo"); + joinableDataSourceNames.add("foo"); + + DataSourceInformation foo = + new DataSourceInformation( + "foo", + RowSignature.builder() + .add("c1", ColumnType.FLOAT) + .add("c2", ColumnType.DOUBLE) + .build() + ); + + DataSourceInformation bar = + new DataSourceInformation( + "bar", + RowSignature.builder() + .add("d1", ColumnType.FLOAT) + .add("d2", ColumnType.DOUBLE) + .build() + ); + + DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build(foo); + Assert.assertTrue(fooDs.isJoinable()); + Assert.assertTrue(fooDs.isBroadcast()); + Assert.assertEquals(fooDs.dataSource().getName(), foo.getDatasource()); + Assert.assertEquals(fooDs.rowSignature(), foo.getRowSignature()); + + DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build(bar); + Assert.assertFalse(barDs.isJoinable()); + Assert.assertFalse(barDs.isBroadcast()); + Assert.assertEquals(barDs.dataSource().getName(), bar.getDatasource()); + Assert.assertEquals(barDs.rowSignature(), bar.getRowSignature()); + } +} From e630b9b730598ed34623e211489345830abfd313 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 15:16:51 +0530 Subject: [PATCH 15/82] Add test for QueryableCoordinatorServerViewTest --- .../QueryableCoordinatorServerViewTest.java | 398 ++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java diff --git a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java new file mode 100644 index 000000000000..0bb391165feb --- /dev/null +++ b/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.smile.SmileFactory; +import com.fasterxml.jackson.dataformat.smile.SmileGenerator; +import com.google.common.base.Function; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.apache.curator.utils.ZKPaths; +import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; +import org.apache.druid.client.selector.RandomServerSelectorStrategy; +import org.apache.druid.curator.CuratorTestBase; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.QueryWatcher; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.initialization.ZkPathsConfig; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.TimelineLookup; +import org.apache.druid.timeline.TimelineObjectHolder; +import org.apache.druid.timeline.partition.NoneShardSpec; +import org.apache.druid.timeline.partition.PartitionHolder; +import org.easymock.EasyMock; +import org.joda.time.Interval; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; + +public class QueryableCoordinatorServerViewTest extends CuratorTestBase +{ + private final ObjectMapper jsonMapper; + private final ZkPathsConfig zkPathsConfig; + private final String inventoryPath; + + private CountDownLatch segmentViewInitLatch; + private CountDownLatch segmentAddedLatch; + private CountDownLatch segmentRemovedLatch; + + private BatchServerInventoryView baseView; + private QueryableCoordinatorServerView overlordServerView; + + public QueryableCoordinatorServerViewTest() + { + jsonMapper = TestHelper.makeJsonMapper(); + zkPathsConfig = new ZkPathsConfig(); + inventoryPath = zkPathsConfig.getLiveSegmentsPath(); + } + + @Before + public void setUp() throws Exception + { + setupServerAndCurator(); + curator.start(); + curator.blockUntilConnected(); + } + + @Test + public void testSingleServerAddedRemovedSegment() throws Exception + { + segmentViewInitLatch = new CountDownLatch(1); + segmentAddedLatch = new CountDownLatch(1); + segmentRemovedLatch = new CountDownLatch(1); + + setupViews(); + + final DruidServer druidServer = new DruidServer( + "localhost:1234", + "localhost:1234", + null, + 10000000L, + ServerType.HISTORICAL, + "default_tier", + 0 + ); + + setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper); + + final DataSegment segment = dataSegmentWithIntervalAndVersion("2014-10-20T00:00:00Z/P1D", "v1"); + final int partition = segment.getShardSpec().getPartitionNum(); + final Interval intervals = Intervals.of("2014-10-20T00:00:00Z/P1D"); + announceSegmentForServer(druidServer, segment, zkPathsConfig, jsonMapper); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); + + TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + List serverLookupRes = (List) timeline.lookup( + intervals + ); + Assert.assertEquals(1, serverLookupRes.size()); + + TimelineObjectHolder actualTimelineObjectHolder = serverLookupRes.get(0); + Assert.assertEquals(intervals, actualTimelineObjectHolder.getInterval()); + Assert.assertEquals("v1", actualTimelineObjectHolder.getVersion()); + + PartitionHolder actualPartitionHolder = actualTimelineObjectHolder.getObject(); + Assert.assertTrue(actualPartitionHolder.isComplete()); + Assert.assertEquals(1, Iterables.size(actualPartitionHolder)); + + SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject(); + Assert.assertFalse(segmentLoadInfo.isEmpty()); + Assert.assertEquals( + druidServer.getMetadata(), + Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers()) + ); + Assert.assertNotNull(timeline.findChunk(intervals, "v1", partition)); + + unannounceSegmentForServer(druidServer, segment); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + + Assert.assertEquals( + 0, + ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() + ); + Assert.assertNull(timeline.findChunk(intervals, "v1", partition)); + } + + @Test + public void testMultipleServerAddedRemovedSegment() throws Exception + { + segmentViewInitLatch = new CountDownLatch(1); + segmentAddedLatch = new CountDownLatch(5); + + // temporarily set latch count to 1 + segmentRemovedLatch = new CountDownLatch(1); + + setupViews(); + + final List druidServers = Lists.transform( + ImmutableList.of("localhost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), + new Function() + { + @Override + public DruidServer apply(String input) + { + return new DruidServer( + input, + input, + null, + 10000000L, + ServerType.HISTORICAL, + "default_tier", + 0 + ); + } + } + ); + + for (DruidServer druidServer : druidServers) { + setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper); + } + + final List segments = Lists.transform( + ImmutableList.of( + Pair.of("2011-04-01/2011-04-03", "v1"), + Pair.of("2011-04-03/2011-04-06", "v1"), + Pair.of("2011-04-01/2011-04-09", "v2"), + Pair.of("2011-04-06/2011-04-09", "v3"), + Pair.of("2011-04-01/2011-04-02", "v3") + ), new Function, DataSegment>() + { + @Override + public DataSegment apply(Pair input) + { + return dataSegmentWithIntervalAndVersion(input.lhs, input.rhs); + } + } + ); + + for (int i = 0; i < 5; ++i) { + announceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig, jsonMapper); + } + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); + + TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + assertValues( + Arrays.asList( + createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), + createExpected("2011-04-02/2011-04-06", "v2", druidServers.get(2), segments.get(2)), + createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3)) + ), + (List) timeline.lookup( + Intervals.of( + "2011-04-01/2011-04-09" + ) + ) + ); + + // unannounce the segment created by dataSegmentWithIntervalAndVersion("2011-04-01/2011-04-09", "v2") + unannounceSegmentForServer(druidServers.get(2), segments.get(2)); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + + // renew segmentRemovedLatch since we still have 4 segments to unannounce + segmentRemovedLatch = new CountDownLatch(4); + + timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + assertValues( + Arrays.asList( + createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), + createExpected("2011-04-02/2011-04-03", "v1", druidServers.get(0), segments.get(0)), + createExpected("2011-04-03/2011-04-06", "v1", druidServers.get(1), segments.get(1)), + createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3)) + ), + (List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")) + ); + + // unannounce all the segments + for (int i = 0; i < 5; ++i) { + // skip the one that was previously unannounced + if (i != 2) { + unannounceSegmentForServer(druidServers.get(i), segments.get(i)); + } + } + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + + Assert.assertEquals( + 0, + ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() + ); + } + + private void unannounceSegmentForServer(DruidServer druidServer, DataSegment segment) throws Exception + { + curator + .delete() + .guaranteed() + .forPath(ZKPaths.makePath(inventoryPath, druidServer.getHost(), segment.getId().toString())); + } + + private Pair>> createExpected( + String intervalStr, + String version, + DruidServer druidServer, + DataSegment segment + ) + { + return Pair.of(Intervals.of(intervalStr), Pair.of(version, Pair.of(druidServer, segment))); + } + + private void assertValues( + List>>> expected, List actual + ) + { + Assert.assertEquals(expected.size(), actual.size()); + + for (int i = 0; i < expected.size(); ++i) { + Pair>> expectedPair = expected.get(i); + TimelineObjectHolder actualTimelineObjectHolder = actual.get(i); + + Assert.assertEquals(expectedPair.lhs, actualTimelineObjectHolder.getInterval()); + Assert.assertEquals(expectedPair.rhs.lhs, actualTimelineObjectHolder.getVersion()); + + PartitionHolder actualPartitionHolder = actualTimelineObjectHolder.getObject(); + Assert.assertTrue(actualPartitionHolder.isComplete()); + Assert.assertEquals(1, Iterables.size(actualPartitionHolder)); + + SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject(); + Assert.assertFalse(segmentLoadInfo.isEmpty()); + Assert.assertEquals(expectedPair.rhs.rhs.lhs.getMetadata(), + Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers())); + } + } + + private void setupViews() throws Exception + { + baseView = new BatchServerInventoryView( + zkPathsConfig, + curator, + jsonMapper, + Predicates.alwaysTrue(), + "test" + ) + { + @Override + public void registerSegmentCallback(Executor exec, final SegmentCallback callback) + { + super.registerSegmentCallback( + exec, + new SegmentCallback() + { + @Override + public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + CallbackAction res = callback.segmentAdded(server, segment); + segmentAddedLatch.countDown(); + return res; + } + + @Override + public CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) + { + CallbackAction res = callback.segmentRemoved(server, segment); + segmentRemovedLatch.countDown(); + return res; + } + + @Override + public CallbackAction segmentViewInitialized() + { + CallbackAction res = callback.segmentViewInitialized(); + segmentViewInitLatch.countDown(); + return res; + } + } + ); + } + }; + + overlordServerView = new QueryableCoordinatorServerView( + EasyMock.createMock(QueryToolChestWarehouse.class), + EasyMock.createMock(QueryWatcher.class), + getSmileMapper(), + EasyMock.createMock(HttpClient.class), + baseView, + new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), + new NoopServiceEmitter(), + new CoordinatorSegmentWatcherConfig() + ); + + baseView.start(); + overlordServerView.start(); + } + + private ObjectMapper getSmileMapper() + { + final SmileFactory smileFactory = new SmileFactory(); + smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false); + smileFactory.delegateToTextual(true); + final ObjectMapper retVal = new DefaultObjectMapper(smileFactory, "broker"); + retVal.getFactory().setCodec(retVal); + return retVal; + } + + + private DataSegment dataSegmentWithIntervalAndVersion(String intervalStr, String version) + { + return DataSegment.builder() + .dataSource("test_overlord_server_view") + .interval(Intervals.of(intervalStr)) + .loadSpec( + ImmutableMap.of( + "type", + "local", + "path", + "somewhere" + ) + ) + .version(version) + .dimensions(ImmutableList.of()) + .metrics(ImmutableList.of()) + .shardSpec(NoneShardSpec.instance()) + .binaryVersion(9) + .size(0) + .build(); + } + + @After + public void tearDown() throws Exception + { + baseView.stop(); + tearDownServerAndCurator(); + } +} From 17c351495a7b895571227fe4f396cbc220b9bd80 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 15:17:12 +0530 Subject: [PATCH 16/82] Add test for BrokerSegmentMetadataCache --- .../BrokerSegmentMetadataCacheTest.java | 1545 +++++++++++++++++ 1 file changed, 1545 insertions(+) create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java new file mode 100644 index 000000000000..eeec64ca296f --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -0,0 +1,1545 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.java.util.metrics.StubServiceEmitter; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.DruidMetrics; +import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; +import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; +import org.apache.druid.query.metadata.metadata.ColumnAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.join.JoinConditionAnalysis; +import org.apache.druid.segment.join.Joinable; +import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.DataSourceInformation; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.TestTimelineServerView; +import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.QueryLifecycle; +import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.QueryResponse; +import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.AllowAllAuthenticator; +import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.sql.calcite.table.DruidTable; +import org.apache.druid.sql.calcite.util.CalciteTestBase; +import org.apache.druid.sql.calcite.util.CalciteTests; +import org.apache.druid.sql.calcite.util.TestDataBuilder; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +// test polling from coordinator and when coordinator doesn't return anything +// test the result +public class BrokerSegmentMetadataCacheTest extends CalciteTestBase +{ + private final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); + // Timeout to allow (rapid) debugging, while not blocking tests with errors. + private static final int WAIT_TIMEOUT_SECS = 6; + + private SpecificSegmentsQuerySegmentWalker walker; + private TestTimelineServerView serverView; + private List druidServers; + private BrokerSegmentMetadataCache runningSchema; + private CountDownLatch buildTableLatch = new CountDownLatch(1); + private CountDownLatch markDataSourceLatch = new CountDownLatch(1); + private CountDownLatch refreshLatch = new CountDownLatch(1); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + SegmentManager segmentManager; + Set segmentDataSourceNames; + Set joinableDataSourceNames; + JoinableFactory globalTableJoinable; + CountDownLatch getDatasourcesLatch = new CountDownLatch(1); + private QueryRunnerFactoryConglomerate conglomerate; + private Closer resourceCloser; + + private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); + static final List ROWS1 = ImmutableList.of( + TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), + TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), + TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) + ); + + static final List ROWS2 = ImmutableList.of( + TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), + TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), + TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) + ); + + @Before + public void setUp() throws Exception + { + segmentDataSourceNames = Sets.newConcurrentHashSet(); + joinableDataSourceNames = Sets.newConcurrentHashSet(); + + segmentManager = new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + { + @Override + public Set getDataSourceNames() + { + getDatasourcesLatch.countDown(); + return segmentDataSourceNames; + } + }; + + globalTableJoinable = new JoinableFactory() + { + @Override + public boolean isDirectlyJoinable(DataSource dataSource) + { + return dataSource instanceof GlobalTableDataSource && + joinableDataSourceNames.contains(((GlobalTableDataSource) dataSource).getName()); + } + + @Override + public Optional build( + DataSource dataSource, + JoinConditionAnalysis condition + ) + { + return Optional.empty(); + } + }; + + resourceCloser = Closer.create(); + conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); + + final File tmpDir = temporaryFolder.newFolder(); + final QueryableIndex index1 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(ROWS1) + .buildMMappedIndex(); + + final QueryableIndex index2 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "2")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new LongSumAggregatorFactory("m1", "m1")) + .withRollup(false) + .build() + ) + .rows(ROWS2) + .buildMMappedIndex(); + + final InputRowSchema rowSchema = new InputRowSchema( + new TimestampSpec("t", null, null), + DimensionsSpec.builder().useSchemaDiscovery(true).build(), + null + ); + final List autoRows1 = ImmutableList.of( + TestDataBuilder.createRow( + ImmutableMap.builder() + .put("t", "2023-01-01T00:00Z") + .put("numbery", 1.1f) + .put("numberyArrays", ImmutableList.of(1L, 2L, 3L)) + .put("stringy", ImmutableList.of("a", "b", "c")) + .put("array", ImmutableList.of(1.1, 2.2, 3.3)) + .put("nested", ImmutableMap.of("x", 1L, "y", 2L)) + .build(), + rowSchema + ) + ); + final List autoRows2 = ImmutableList.of( + TestDataBuilder.createRow( + ImmutableMap.builder() + .put("t", "2023-01-02T00:00Z") + .put("numbery", 1L) + .put("numberyArrays", ImmutableList.of(3.3, 2.2, 3.1)) + .put("stringy", "a") + .put("array", ImmutableList.of(1L, 2L, 3L)) + .put("nested", "hello") + .build(), + rowSchema + ) + ); + + final QueryableIndex indexAuto1 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec(rowSchema.getTimestampSpec()) + .withDimensionsSpec(rowSchema.getDimensionsSpec()) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows1) + .buildMMappedIndex(); + + final QueryableIndex indexAuto2 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec( + new TimestampSpec("t", null, null) + ) + .withDimensionsSpec( + DimensionsSpec.builder().useSchemaDiscovery(true).build() + ) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows2) + .buildMMappedIndex(); + + walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( + DataSegment.builder() + .dataSource(CalciteTests.DATASOURCE1) + .interval(Intervals.of("2000/P1Y")) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index1 + ).add( + DataSegment.builder() + .dataSource(CalciteTests.DATASOURCE1) + .interval(Intervals.of("2001/P1Y")) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index2 + ).add( + DataSegment.builder() + .dataSource(CalciteTests.DATASOURCE2) + .interval(index2.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index2 + ).add( + DataSegment.builder() + .dataSource(CalciteTests.SOME_DATASOURCE) + .interval(Intervals.of("2023-01-01T00Z/P1D")) + .version("1") + .shardSpec(new LinearShardSpec(1)) + .size(0) + .build(), + indexAuto1 + ).add( + DataSegment.builder() + .dataSource(CalciteTests.SOME_DATASOURCE) + .interval(Intervals.of("2023-01-02T00Z/P1D")) + .version("1") + .shardSpec(new LinearShardSpec(1)) + .size(0) + .build(), + indexAuto2 + ); + final DataSegment segment1 = new DataSegment( + "foo3", + Intervals.of("2012/2013"), + "version3", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(2, 3), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + final List realtimeSegments = ImmutableList.of(segment1); + serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); + druidServers = serverView.getDruidServers(); + } + + public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException + { + return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); + } + + public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMetadataCacheConfig config) throws InterruptedException + { + Preconditions.checkState(runningSchema == null); + runningSchema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + config, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public DataSourceInformation buildDruidTable(String dataSource) + { + DataSourceInformation table = super.buildDruidTable(dataSource); + buildTableLatch.countDown(); + return table; + } + + @Override + public void markDataSourceAsNeedRebuild(String datasource) + { + super.markDataSourceAsNeedRebuild(datasource); + markDataSourceLatch.countDown(); + } + }; + + runningSchema.start(); + runningSchema.awaitInitialization(); + return runningSchema; + } + + public BrokerSegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedException + { + Preconditions.checkState(runningSchema == null); + runningSchema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void markDataSourceAsNeedRebuild(String datasource) + { + super.markDataSourceAsNeedRebuild(datasource); + markDataSourceLatch.countDown(); + } + + @Override + @VisibleForTesting + public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws + IOException + { + super.refresh(segmentsToRefresh, dataSourcesToRebuild); + refreshLatch.countDown(); + } + }; + + runningSchema.start(); + runningSchema.awaitInitialization(); + return runningSchema; + } + + @After + public void tearDown() throws Exception + { + if (runningSchema != null) { + runningSchema.stop(); + } + walker.close(); + resourceCloser.close(); + } + + @Test + public void testGetTableMap() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); + + final Set tableNames = schema.getDatasourceNames(); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), tableNames); + } + + @Test + public void testSchemaInit() throws InterruptedException + { + BrokerSegmentMetadataCache schema2 = buildSchemaMarkAndTableLatch(); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema2.getDatasourceNames()); + } + + @Test + public void testGetTableMapFoo() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata("foo"); + final DruidTable fooTable = new DatasourceTable(fooDs); + final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); + final List fields = rowType.getFieldList(); + + Assert.assertEquals(6, fields.size()); + + Assert.assertEquals("__time", fields.get(0).getName()); + Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + + Assert.assertEquals("dim2", fields.get(1).getName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName()); + + Assert.assertEquals("m1", fields.get(2).getName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getSqlTypeName()); + + Assert.assertEquals("dim1", fields.get(3).getName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName()); + + Assert.assertEquals("cnt", fields.get(4).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getSqlTypeName()); + + Assert.assertEquals("unique_dim1", fields.get(5).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); + } + + @Test + public void testGetTableMapFoo2() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata("foo2"); + final DruidTable fooTable = new DatasourceTable(fooDs); + final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); + final List fields = rowType.getFieldList(); + + Assert.assertEquals(3, fields.size()); + + Assert.assertEquals("__time", fields.get(0).getName()); + Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + + Assert.assertEquals("dim2", fields.get(1).getName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName()); + + Assert.assertEquals("m1", fields.get(2).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(2).getType().getSqlTypeName()); + } + + @Test + public void testGetTableMapSomeTable() throws InterruptedException + { + // using 'newest first' column type merge strategy, the types are expected to be the types defined in the newer + // segment, except for json, which is special handled + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch( + new BrokerSegmentMetadataCacheConfig() { + @Override + public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + { + return new SegmentMetadataCache.FirstTypeMergePolicy(); + } + } + ); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); + final DruidTable table = new DatasourceTable(fooDs); + final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); + final List fields = rowType.getFieldList(); + + Assert.assertEquals(9, fields.size()); + + Assert.assertEquals("__time", fields.get(0).getName()); + Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + + Assert.assertEquals("numbery", fields.get(1).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(1).getType().getSqlTypeName()); + + Assert.assertEquals("numberyArrays", fields.get(2).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(2).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("stringy", fields.get(3).getName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName()); + + Assert.assertEquals("array", fields.get(4).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(4).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("nested", fields.get(5).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); + + Assert.assertEquals("cnt", fields.get(6).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(6).getType().getSqlTypeName()); + + Assert.assertEquals("m1", fields.get(7).getName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(7).getType().getSqlTypeName()); + + Assert.assertEquals("unique_dim1", fields.get(8).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(8).getType().getSqlTypeName()); + } + + @Test + public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws InterruptedException + { + // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the + // least restrictive blend across all segments + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); + final DruidTable table = new DatasourceTable(fooDs); + final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); + final List fields = rowType.getFieldList(); + + Assert.assertEquals(9, fields.size()); + + Assert.assertEquals("__time", fields.get(0).getName()); + Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + + Assert.assertEquals("numbery", fields.get(1).getName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(1).getType().getSqlTypeName()); + + Assert.assertEquals("numberyArrays", fields.get(2).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(2).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("stringy", fields.get(3).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(3).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("array", fields.get(4).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(4).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(4).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("nested", fields.get(5).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); + + Assert.assertEquals("cnt", fields.get(6).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(6).getType().getSqlTypeName()); + + Assert.assertEquals("m1", fields.get(7).getName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(7).getType().getSqlTypeName()); + + Assert.assertEquals("unique_dim1", fields.get(8).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(8).getType().getSqlTypeName()); + } + + /** + * This tests that {@link AvailableSegmentMetadata#getNumRows()} is correct in case + * of multiple replicas i.e. when {@link SegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} + * is called more than once for same segment + * @throws InterruptedException + */ + @Test + public void testAvailableSegmentMetadataNumRows() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); + final List segments = segmentsMetadata.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // find the only segment with datasource "foo2" + final DataSegment existingSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(existingSegment); + final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); + // update AvailableSegmentMetadata of existingSegment with numRows=5 + AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); + schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); + // find a druidServer holding existingSegment + final Pair pair = druidServers + .stream() + .flatMap(druidServer -> druidServer + .iterateAllSegments() + .stream() + .filter(segment -> segment.getId().equals(existingSegment.getId())) + .map(segment -> Pair.of(druidServer, segment)) + ) + .findAny() + .orElse(null); + Assert.assertNotNull(pair); + final ImmutableDruidServer server = pair.lhs; + Assert.assertNotNull(server); + final DruidServerMetadata druidServerMetadata = server.getMetadata(); + // invoke SegmentMetadataCache#addSegment on existingSegment + schema.addSegment(druidServerMetadata, existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + // get the only segment with datasource "foo2" + final DataSegment currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); + Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); + // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before + Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); + } + + @Test + public void testNullDatasource() throws IOException, InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // segments contains two segments with datasource "foo" and one with datasource "foo2" + // let's remove the only segment with datasource "foo2" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + } + + @Test + public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // remove one of the segments with datasource "foo" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + } + + @Test + public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); + final List segments = segmentsMetadata.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + // find the only realtime segment with datasource "foo3" + final DataSegment existingSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo3")) + .findFirst() + .orElse(null); + Assert.assertNotNull(existingSegment); + final AvailableSegmentMetadata metadata = segmentsMetadata.get(existingSegment.getId()); + Assert.assertEquals(1L, metadata.isRealtime()); + // get the historical server + final ImmutableDruidServer historicalServer = druidServers.stream() + .filter(s -> s.getType().equals(ServerType.HISTORICAL)) + .findAny() + .orElse(null); + + Assert.assertNotNull(historicalServer); + final DruidServerMetadata historicalServerMetadata = historicalServer.getMetadata(); + + // add existingSegment to historical + schema.addSegment(historicalServerMetadata, existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + // get the segment with datasource "foo3" + DataSegment currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo3")) + .findFirst() + .orElse(null); + Assert.assertNotNull(currentSegment); + AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(0L, currentMetadata.isRealtime()); + + ImmutableDruidServer realtimeServer = druidServers.stream() + .filter(s -> s.getType().equals(ServerType.REALTIME)) + .findAny() + .orElse(null); + Assert.assertNotNull(realtimeServer); + // drop existingSegment from realtime task + schema.removeServerSegment(realtimeServer.getMetadata(), existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo3")) + .findFirst() + .orElse(null); + Assert.assertNotNull(currentSegment); + currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(0L, currentMetadata.isRealtime()); + } + + @Test + public void testSegmentAddedCallbackAddNewHistoricalSegment() throws InterruptedException + { + String datasource = "newSegmentAddTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + SegmentMetadataCache schema = new SegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + }; + + serverView.addSegment(newSegment(datasource, 1), ServerType.HISTORICAL); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + AvailableSegmentMetadata metadata = metadatas.get(0); + Assert.assertEquals(0, metadata.isRealtime()); + Assert.assertEquals(0, metadata.getNumRows()); + Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); + } + + @Test + public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedException + { + String datasource = "newSegmentAddTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(2); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + }; + + DataSegment segment = newSegment(datasource, 1); + serverView.addSegment(segment, ServerType.REALTIME); + serverView.addSegment(segment, ServerType.HISTORICAL); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + AvailableSegmentMetadata metadata = metadatas.get(0); + Assert.assertEquals(0, metadata.isRealtime()); // realtime flag is unset when there is any historical + Assert.assertEquals(0, metadata.getNumRows()); + Assert.assertEquals(2, metadata.getNumReplicas()); + Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); + Assert.assertFalse(schema.getMutableSegments().contains(metadata.getSegment().getId())); + } + + @Test + public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedException + { + String datasource = "newSegmentAddTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + }; + + serverView.addSegment(newSegment(datasource, 1), ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + AvailableSegmentMetadata metadata = metadatas.get(0); + Assert.assertEquals(1, metadata.isRealtime()); + Assert.assertEquals(0, metadata.getNumRows()); + Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); + Assert.assertTrue(schema.getMutableSegments().contains(metadata.getSegment().getId())); + } + + @Test + public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedException + { + String datasource = "newSegmentAddTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + }; + + serverView.addSegment(newSegment(datasource, 1), ServerType.BROKER); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(6, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(0, metadatas.size()); + Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); + } + + @Test + public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws InterruptedException, IOException + { + String datasource = "segmentRemoveTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + CountDownLatch removeSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeSegment(final DataSegment segment) + { + super.removeSegment(segment); + if (datasource.equals(segment.getDataSource())) { + removeSegmentLatch.countDown(); + } + } + }; + + DataSegment segment = newSegment(datasource, 1); + serverView.addSegment(segment, ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(Sets.newHashSet(segment.getId()), Sets.newHashSet(datasource)); + + serverView.removeSegment(segment, ServerType.REALTIME); + Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(6, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(0, metadatas.size()); + Assert.assertFalse(schema.getSegmentsNeedingRefresh().contains(segment.getId())); + Assert.assertFalse(schema.getMutableSegments().contains(segment.getId())); + Assert.assertFalse(schema.getDataSourcesNeedingRebuild().contains(datasource)); + Assert.assertFalse(schema.getDatasourceNames().contains(datasource)); + } + + @Test + public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws InterruptedException, IOException + { + String datasource = "segmentRemoveTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(2); + CountDownLatch removeSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeSegment(final DataSegment segment) + { + super.removeSegment(segment); + if (datasource.equals(segment.getDataSource())) { + removeSegmentLatch.countDown(); + } + } + }; + + List segments = ImmutableList.of( + newSegment(datasource, 1), + newSegment(datasource, 2) + ); + serverView.addSegment(segments.get(0), ServerType.REALTIME); + serverView.addSegment(segments.get(1), ServerType.HISTORICAL); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); + + serverView.removeSegment(segments.get(0), ServerType.REALTIME); + Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + Assert.assertFalse(schema.getSegmentsNeedingRefresh().contains(segments.get(0).getId())); + Assert.assertFalse(schema.getMutableSegments().contains(segments.get(0).getId())); + Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); + Assert.assertTrue(schema.getDatasourceNames().contains(datasource)); + } + + @Test + public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws InterruptedException + { + String datasource = "serverSegmentRemoveTest"; + CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.removeServerSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + removeServerSegmentLatch.countDown(); + } + } + }; + + serverView.addSegment(newSegment(datasource, 1), ServerType.BROKER); + + serverView.removeSegment(newSegment(datasource, 1), ServerType.HISTORICAL); + Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(6, schema.getTotalSegments()); + } + + @Test + public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws InterruptedException + { + String datasource = "serverSegmentRemoveTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.removeServerSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + removeServerSegmentLatch.countDown(); + } + } + }; + + DataSegment segment = newSegment(datasource, 1); + serverView.addSegment(segment, ServerType.HISTORICAL); + serverView.addSegment(segment, ServerType.BROKER); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + serverView.removeSegment(segment, ServerType.BROKER); + Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); + } + + @Test + public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws InterruptedException + { + String datasource = "serverSegmentRemoveTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.removeServerSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + removeServerSegmentLatch.countDown(); + } + } + }; + + DataSegment segment = newSegment(datasource, 1); + serverView.addSegment(segment, ServerType.HISTORICAL); + serverView.addSegment(segment, ServerType.BROKER); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + serverView.removeSegment(segment, ServerType.HISTORICAL); + Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + AvailableSegmentMetadata metadata = metadatas.get(0); + Assert.assertEquals(0, metadata.isRealtime()); + Assert.assertEquals(0, metadata.getNumRows()); + Assert.assertEquals(0, metadata.getNumReplicas()); // brokers are not counted as replicas yet + } + + /** + * Test actions on the cache. The current design of the cache makes testing far harder + * than it should be. + * + * - The cache is refreshed on a schedule. + * - Datasources are added to the refresh queue via an unsynchronized thread. + * - The refresh loop always refreshes since one of the segments is dynamic. + * + * The use of latches tries to keep things synchronized, but there are many + * moving parts. A simpler technique is sorely needed. + */ + @Test + public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws InterruptedException + { + BrokerSegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + + markDataSourceLatch = new CountDownLatch(1); + refreshLatch = new CountDownLatch(1); + final DataSegment someNewBrokerSegment = new DataSegment( + "foo", + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(2, 3), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); + Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for build twice + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) + refreshLatch = new CountDownLatch(1); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + + fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + + // now remove it + markDataSourceLatch = new CountDownLatch(1); + refreshLatch = new CountDownLatch(1); + serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); + + Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for build twice + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) + refreshLatch = new CountDownLatch(1); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + + fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + } + + @Test + public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + + markDataSourceLatch = new CountDownLatch(1); + refreshLatch = new CountDownLatch(1); + final DataSegment someNewBrokerSegment = new DataSegment( + "foo", + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(2, 3), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); + + Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for build twice + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) + refreshLatch = new CountDownLatch(1); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + + fooTable = schema.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + + // now remove it + markDataSourceLatch = new CountDownLatch(1); + refreshLatch = new CountDownLatch(1); + serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); + + Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for build twice + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) + refreshLatch = new CountDownLatch(1); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + + fooTable = schema.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + } + + /** + * Ensure that the BrokerInternalQueryConfig context is honored for this internally generated SegmentMetadata Query + */ + @Test + public void testRunSegmentMetadataQueryWithContext() throws Exception + { + Map queryContext = ImmutableMap.of( + QueryContexts.PRIORITY_KEY, 5, + QueryContexts.BROKER_PARALLEL_MERGE_KEY, false + ); + + String brokerInternalQueryConfigJson = "{\"context\": { \"priority\": 5} }"; + + TestHelper.makeJsonMapper(); + InternalQueryConfig internalQueryConfig = MAPPER.readValue( + MAPPER.writeValueAsString( + MAPPER.readValue(brokerInternalQueryConfigJson, InternalQueryConfig.class) + ), + InternalQueryConfig.class + ); + + DataSegment segment = newSegment("test", 0); + List segmentIterable = ImmutableList.of(segment.getId()); + + // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. + SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(segment.getDataSource()), + new MultipleSpecificSegmentSpec( + segmentIterable.stream() + .map(SegmentId::toDescriptor).collect(Collectors.toList())), + new AllColumnIncluderator(), + false, + queryContext, + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); + QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); + + // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. + BrokerSegmentMetadataCache mySchema = new BrokerSegmentMetadataCache( + factoryMock, + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + internalQueryConfig, + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ); + + EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); + // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context + EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) + .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); + + EasyMock.replay(factoryMock, lifecycleMock); + + mySchema.runSegmentMetadataQuery(segmentIterable); + + EasyMock.verify(factoryMock, lifecycleMock); + + } + + @Test + public void testSegmentMetadataColumnType() + { + // Verify order is preserved. + final LinkedHashMap columns = new LinkedHashMap<>(); + columns.put( + "a", + new ColumnAnalysis(ColumnType.STRING, ColumnType.STRING.asTypeString(), false, true, 1234, 26, "a", "z", null) + ); + + columns.put( + "count", + new ColumnAnalysis(ColumnType.LONG, ColumnType.LONG.asTypeString(), false, true, 1234, 26, "a", "z", null) + ); + + columns.put( + "b", + new ColumnAnalysis(ColumnType.DOUBLE, ColumnType.DOUBLE.asTypeString(), false, true, 1234, 26, null, null, null) + ); + + RowSignature signature = SegmentMetadataCache.analysisToRowSignature( + new SegmentAnalysis( + "id", + ImmutableList.of(Intervals.utc(1L, 2L)), + columns, + 1234, + 100, + null, + null, + null, + null + ) + ); + + Assert.assertEquals( + RowSignature.builder() + .add("a", ColumnType.STRING) + .add("count", ColumnType.LONG) + .add("b", ColumnType.DOUBLE) + .build(), + signature + ); + } + + @Test + public void testSegmentMetadataFallbackType() + { + RowSignature signature = SegmentMetadataCache.analysisToRowSignature( + new SegmentAnalysis( + "id", + ImmutableList.of(Intervals.utc(1L, 2L)), + new LinkedHashMap<>( + ImmutableMap.of( + "a", + new ColumnAnalysis( + null, + ColumnType.STRING.asTypeString(), + false, + true, + 1234, + 26, + "a", + "z", + null + ), + "count", + new ColumnAnalysis( + null, + ColumnType.LONG.asTypeString(), + false, + true, + 1234, + 26, + "a", + "z", + null + ) + ) + ), + 1234, + 100, + null, + null, + null, + null + ) + ); + Assert.assertEquals( + RowSignature.builder().add("a", ColumnType.STRING).add("count", ColumnType.LONG).build(), + signature + ); + } + + @Test + public void testStaleDatasourceRefresh() throws IOException, InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Set segments = new HashSet<>(); + Set datasources = new HashSet<>(); + datasources.add("wat"); + Assert.assertNull(schema.getDatasource("wat")); + schema.refresh(segments, datasources); + Assert.assertNull(schema.getDatasource("wat")); + } + + @Test + public void testRefreshShouldEmitMetrics() throws InterruptedException, IOException + { + String datasource = "xyz"; + CountDownLatch addSegmentLatch = new CountDownLatch(2); + StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + emitter, + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeSegment(final DataSegment segment) + { + super.removeSegment(segment); + } + }; + + List segments = ImmutableList.of( + newSegment(datasource, 1), + newSegment(datasource, 2) + ); + serverView.addSegment(segments.get(0), ServerType.HISTORICAL); + serverView.addSegment(segments.get(1), ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); + + emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); + emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); + } + + private static DataSegment newSegment(String datasource, int partitionId) + { + return new DataSegment( + datasource, + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(partitionId, 0), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + } +} From f0baf33d55896155f7543d8f12242593d24b7e29 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 15:17:30 +0530 Subject: [PATCH 17/82] minor code changes and fix checkstyle issues --- ...ruidSchemaInternRowSignatureBenchmark.java | 9 ++--- .../druid/benchmark/query/SqlBenchmark.java | 2 +- .../query/SqlExpressionBenchmark.java | 2 +- .../query/SqlNestedDataBenchmark.java | 2 +- .../benchmark/query/SqlVsNativeBenchmark.java | 2 +- ...ressedBigDecimalSqlAggregatorTestBase.java | 2 +- .../sql/TDigestSketchSqlAggregatorTest.java | 2 +- .../hll/sql/HllSketchSqlAggregatorTest.java | 2 +- .../sql/DoublesSketchSqlAggregatorTest.java | 2 +- .../sql/ThetaSketchSqlAggregatorTest.java | 2 +- ...ArrayOfDoublesSketchSqlAggregatorTest.java | 2 +- .../sql/BloomFilterSqlAggregatorTest.java | 2 +- ...etsHistogramQuantileSqlAggregatorTest.java | 2 +- .../sql/QuantileSqlAggregatorTest.java | 2 +- .../apache/druid/msq/test/MSQTestBase.java | 2 +- .../sql/VarianceSqlAggregatorTest.java | 2 +- .../apache/druid/client/BrokerServerView.java | 2 -- .../coordinator/CoordinatorClientImpl.java | 3 +- .../metadata/DataSourceInformation.java | 19 ++++++++++ .../metadata/SegmentMetadataCache.java | 18 +++++----- .../metadata/SegmentMetadataCacheConfig.java | 8 +++++ .../server/http/DataSourcesResource.java | 2 +- .../SegmentDataCacheConcurrencyTest.java | 2 +- .../metadata/SegmentMetadataCacheCommon.java | 4 +-- .../SegmentMetadataCacheConfigTest.java | 2 -- .../metadata/SegmentMetadataCacheTest.java | 35 +++++++++---------- .../server/http/MetadataResourceTest.java | 13 +++---- .../java/org/apache/druid/cli/CliBroker.java | 6 ++-- .../org/apache/druid/cli/CliCoordinator.java | 2 +- .../schema/BrokerSegmentMetadataCache.java | 3 +- .../BrokerSegmentMetadataCacheConfig.java | 10 ++++++ .../schema/DruidCalciteSchemaModule.java | 2 -- .../druid/sql/calcite/schema/DruidSchema.java | 2 -- .../apache/druid/sql/SqlStatementTest.java | 2 +- .../sql/avatica/DruidAvaticaHandlerTest.java | 2 +- .../druid/sql/avatica/DruidStatementTest.java | 2 +- .../sql/calcite/BaseCalciteQueryTest.java | 2 +- .../calcite/CalciteNestedDataQueryTest.java | 2 +- .../SqlVectorizedExpressionSanityTest.java | 2 +- .../schema/DruidSchemaNoDataInitTest.java | 5 ++- ...PhysicalDataSourceMetadataBuilderTest.java | 3 +- .../sql/calcite/schema/SystemSchemaTest.java | 1 - .../sql/calcite/util/QueryFrameworkUtils.java | 4 +-- .../sql/calcite/util/SqlTestFramework.java | 2 +- .../sql/calcite/util/TestDataBuilder.java | 2 +- .../druid/sql/http/SqlResourceTest.java | 2 +- 46 files changed, 112 insertions(+), 91 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 4d2d11ba1f3f..5f0634f283a4 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -30,16 +30,13 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.SegmentManager; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Escalator; -import org.apache.druid.sql.calcite.planner.PlannerConfig; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.LinearShardSpec; @@ -104,7 +101,7 @@ public void addSegment(final DruidServerMetadata server, final DataSegment segme } @Override - protected Sequence runSegmentMetadataQuery(Iterable segments) + public Sequence runSegmentMetadataQuery(Iterable segments) { final int numColumns = 1000; LinkedHashMap columnToAnalysisMap = new LinkedHashMap<>(); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index d340f10e35e3..9bfe925ccf32 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -46,6 +46,7 @@ import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.aggregation.ApproxCountDistinctSqlAggregator; @@ -63,7 +64,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 15c26f1e4577..dbdd4ea19620 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -36,6 +36,7 @@ import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest; @@ -48,7 +49,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java index 7fc8a71c3fd3..1915776dbca3 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java @@ -45,6 +45,7 @@ import org.apache.druid.segment.transform.ExpressionTransform; import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest; @@ -57,7 +58,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java index 8c709a7457f9..e4f8b5570bf0 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java @@ -38,6 +38,7 @@ import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.planner.CalciteRulesManager; @@ -49,7 +50,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java index b6aa0d866e5d..cda55e314e0b 100644 --- a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java +++ b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java @@ -41,10 +41,10 @@ import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java index dd15a6defac2..476b7734fca9 100644 --- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java @@ -46,10 +46,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java index c08bc8f04345..6f69237546e3 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java @@ -73,10 +73,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.DataSegment; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java index 68f53687f892..e85c49a5274b 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java @@ -57,10 +57,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java index b4d7e29661cb..d5ae2d63f8b1 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java @@ -59,10 +59,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.DataSegment; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java index f262accfef94..77031dbb02ab 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java @@ -46,10 +46,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java index 30b76e8602b5..93902b0996ee 100644 --- a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java @@ -47,10 +47,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java index fea3d871e938..5b529e188e10 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java @@ -49,10 +49,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index 0c81b4c40fcb..b9d4000b6f84 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -48,10 +48,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java index 7c1d99fea5fa..68acd4744d3c 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java @@ -143,6 +143,7 @@ import org.apache.druid.segment.realtime.appenderator.AppenderatorsManager; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DataSegmentAnnouncer; import org.apache.druid.server.coordination.NoopDataSegmentAnnouncer; import org.apache.druid.server.security.AuthConfig; @@ -164,7 +165,6 @@ import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.LookylooModule; import org.apache.druid.sql.calcite.util.QueryFrameworkUtils; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.view.InProcessViewManager; import org.apache.druid.sql.guice.SqlBindings; diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java index 23d404354701..523b7128a0f7 100644 --- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java @@ -55,10 +55,10 @@ import org.apache.druid.segment.serde.ComplexMetrics; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index 5285b243f158..77fc59219096 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import com.google.inject.Inject; import org.apache.druid.client.selector.QueryableDruidServer; @@ -54,7 +53,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index f257a71de5b4..dfe3099c8cf5 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -114,8 +114,7 @@ public ListenableFuture> fetchUsedSegments(String dataSource, public ListenableFuture> fetchDataSourceInformation(Set dataSources) { final String path = "/druid/coordinator/v1/metadata/dataSourceInformation"; - if (null == dataSources) - { + if (null == dataSources) { dataSources = new HashSet<>(); } return FutureUtils.transform( diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index 6501704bddac..2e3257e100c1 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.segment.metadata; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 704fc52de5f8..e5709fac0dbd 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -470,7 +470,7 @@ public Set getDatasourceNames() } @VisibleForTesting - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { // Get lock first so that we won't wait in ConcurrentMap.compute(). synchronized (lock) { @@ -542,7 +542,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @VisibleForTesting - void removeSegment(final DataSegment segment) + public void removeSegment(final DataSegment segment) { // Get lock first so that we won't wait in ConcurrentMap.compute(). synchronized (lock) { @@ -580,7 +580,7 @@ void removeSegment(final DataSegment segment) } @VisibleForTesting - void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) { // Get lock first so that we won't wait in ConcurrentMap.compute(). synchronized (lock) { @@ -664,7 +664,7 @@ private void unmarkSegmentAsMutable(SegmentId segmentId) } @VisibleForTesting - void markDataSourceAsNeedRebuild(String datasource) + public void markDataSourceAsNeedRebuild(String datasource) { synchronized (lock) { dataSourcesNeedingRebuild.add(datasource); @@ -877,7 +877,7 @@ public int getTotalSegments() } @VisibleForTesting - TreeSet getSegmentsNeedingRefresh() + public TreeSet getSegmentsNeedingRefresh() { synchronized (lock) { return segmentsNeedingRefresh; @@ -885,7 +885,7 @@ TreeSet getSegmentsNeedingRefresh() } @VisibleForTesting - TreeSet getMutableSegments() + public TreeSet getMutableSegments() { synchronized (lock) { return mutableSegments; @@ -893,7 +893,7 @@ TreeSet getMutableSegments() } @VisibleForTesting - Set getDataSourcesNeedingRebuild() + public Set getDataSourcesNeedingRebuild() { synchronized (lock) { return dataSourcesNeedingRebuild; @@ -907,7 +907,7 @@ Set getDataSourcesNeedingRebuild() * @return {@link Sequence} of {@link SegmentAnalysis} objects */ @VisibleForTesting - protected Sequence runSegmentMetadataQuery( + public Sequence runSegmentMetadataQuery( final Iterable segments ) { @@ -979,7 +979,7 @@ public static RowSignature analysisToRowSignature(final SegmentAnalysis analysis * This method is not thread-safe and must be used only in unit tests. */ @VisibleForTesting - void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) + public void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) { final ConcurrentSkipListMap dataSourceSegments = segmentMetadataInfo .computeIfAbsent( diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 4cf8d0fb0019..2ebcb0e53c0a 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -20,12 +20,14 @@ package org.apache.druid.segment.metadata; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import org.joda.time.Period; public class SegmentMetadataCacheConfig { @JsonProperty private boolean awaitInitializationOnStart = false; + @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); @JsonProperty @@ -49,6 +51,12 @@ public static SegmentMetadataCacheConfig create( return config; } + @VisibleForTesting + public void setMetadataRefreshPeriod(Period metadataRefreshPeriod) + { + this.metadataRefreshPeriod = metadataRefreshPeriod; + } + public boolean isAwaitInitializationOnStart() { return awaitInitializationOnStart; diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 1cbaa92ae231..c8e998e72cfa 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,9 +30,9 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; +import org.apache.druid.client.CoordinatorTimeline; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; -import org.apache.druid.client.CoordinatorTimeline; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableSegmentLoadInfo; import org.apache.druid.client.SegmentLoadInfo; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 60421be19810..1746d36b3e16 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -22,12 +22,12 @@ import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.FilteringSegmentCallback; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ServerView.CallbackAction; import org.apache.druid.client.ServerView.SegmentCallback; import org.apache.druid.client.ServerView.ServerRemovedCallback; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 0bd83f88f2d2..897f9f0a6463 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -42,7 +42,6 @@ import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -122,7 +121,8 @@ InputRow createRow(final ImmutableMap map, InputRowSchema inputRowSch return MapInputRowParser.parse(inputRowSchema, (Map) map); } - QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) { + QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) + { return new QueryLifecycleFactory( queryToolChestWarehouse, walker, diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java index cf0511dd6f8a..d625229972fe 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java @@ -24,8 +24,6 @@ import org.apache.druid.guice.GuiceInjectors; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; -import org.apache.druid.segment.metadata.SegmentMetadataCache; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.joda.time.Period; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index aa9df38b9b71..1237aced8254 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -296,7 +296,7 @@ public DataSourceInformation buildDruidTable(String dataSource) } @Override - void markDataSourceAsNeedRebuild(String datasource) + public void markDataSourceAsNeedRebuild(String datasource) { super.markDataSourceAsNeedRebuild(datasource); markDataSourceLatch.countDown(); @@ -408,8 +408,6 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); } - - /** * This tests that {@link AvailableSegmentMetadata#getNumRows()} is correct in case * of multiple replicas i.e. when {@link SegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} @@ -585,7 +583,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -626,7 +624,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -671,7 +669,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -713,7 +711,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -752,7 +750,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -761,7 +759,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeSegment(final DataSegment segment) + public void removeSegment(final DataSegment segment) { super.removeSegment(segment); if (datasource.equals(segment.getDataSource())) { @@ -808,7 +806,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -817,7 +815,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeSegment(final DataSegment segment) + public void removeSegment(final DataSegment segment) { super.removeSegment(segment); if (datasource.equals(segment.getDataSource())) { @@ -867,7 +865,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr ) { @Override - void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) { super.removeServerSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -900,7 +898,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -909,7 +907,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) { super.removeServerSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -946,7 +944,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -955,7 +953,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) { super.removeServerSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -1207,7 +1205,6 @@ public void testSegmentMetadataColumnType() ); } - @Test public void testSegmentMetadataFallbackType() { @@ -1285,7 +1282,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -1294,7 +1291,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeSegment(final DataSegment segment) + public void removeSegment(final DataSegment segment) { super.removeSegment(segment); } diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index 141c0ddc5c39..e122334946e5 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -131,10 +131,10 @@ public void testGetAllSegmentsWithOvershadowedStatus() final List resultList = extractResponseList(response); Assert.assertEquals(resultList.size(), 4); Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, null, false), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, null, false), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, null, false), resultList.get(1)); Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, null, false), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); } @Test @@ -228,16 +228,17 @@ public void testGetAllSegmentsIncludingRealtime() final List resultList = extractResponseList(response); Assert.assertEquals(resultList.size(), 6); Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 20L, false), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 30L, false), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 30L, false), resultList.get(1)); Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, null, false), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[0], false, null, 10L, true), resultList.get(4)); - Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[1], false, null, 40L, true), resultList.get(5)); + Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[1], false, null, 40L, true), resultList.get(5)); } @Test - public void testGetDataSourceInformation() { + public void testGetDataSourceInformation() + { SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); Map dataSourceInformationMap = new HashMap<>(); diff --git a/services/src/main/java/org/apache/druid/cli/CliBroker.java b/services/src/main/java/org/apache/druid/cli/CliBroker.java index 854560540664..ba8bd1234eb4 100644 --- a/services/src/main/java/org/apache/druid/cli/CliBroker.java +++ b/services/src/main/java/org/apache/druid/cli/CliBroker.java @@ -26,11 +26,11 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.name.Names; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.HttpServerInventoryViewResource; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.cache.CacheConfig; import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; @@ -71,7 +71,7 @@ import org.apache.druid.server.metrics.QueryCountStatsProvider; import org.apache.druid.server.metrics.SubqueryCountStatsProvider; import org.apache.druid.server.router.TieredBrokerConfig; -import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; +import org.apache.druid.sql.calcite.schema.MetadataSegmentView; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.PruneLoadSpec; import org.eclipse.jetty.server.Server; @@ -129,7 +129,7 @@ protected List getModules() binder.bind(CachingClusteredClient.class).in(LazySingleton.class); LifecycleModule.register(binder, BrokerServerView.class); - LifecycleModule.register(binder, BrokerSegmentMetadataView.class); + LifecycleModule.register(binder, MetadataSegmentView.class); binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); JsonConfigProvider.bind(binder, "druid.broker.cache", CacheConfig.class); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index c61ef1c6c70d..7a3833736cdd 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -40,8 +40,8 @@ import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; import org.apache.druid.client.CoordinatorServerView; -import org.apache.druid.client.HttpServerInventoryViewResource; import org.apache.druid.client.CoordinatorTimeline; +import org.apache.druid.client.HttpServerInventoryViewResource; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.QueryableCoordinatorServerView; import org.apache.druid.client.TimelineServerView; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 8a037e7cd039..8c2369aaab62 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -96,7 +96,8 @@ public void refresh(final Set segmentsToRefresh, final Set da item.getDatasource(), physicalDatasourceMetadataBuilder.build(item) )); - } catch (Exception e) { + } + catch (Exception e) { log.warn(e, "Exception querying coordinator to fetch dataSourceInformation."); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 2554753f79c3..a51aea811e93 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.joda.time.Period; public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig { @@ -38,6 +39,15 @@ public static BrokerSegmentMetadataCacheConfig create() return new BrokerSegmentMetadataCacheConfig(); } + public static BrokerSegmentMetadataCacheConfig create( + String metadataRefreshPeriod + ) + { + BrokerSegmentMetadataCacheConfig config = new BrokerSegmentMetadataCacheConfig(); + config.setMetadataRefreshPeriod(new Period(metadataRefreshPeriod)); + return config; + } + public boolean isMetadataSegmentCacheEnable() { return metadataSegmentCacheEnable; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index 132f2cb92590..26d038ef4a3b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -29,8 +29,6 @@ import org.apache.druid.guice.LifecycleModule; import org.apache.druid.sql.guice.SqlBindings; -import java.util.Properties; - /** * The module responsible for providing bindings to Calcite schemas. */ diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index aa1e7965c27e..69b96403aa82 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -20,11 +20,9 @@ package org.apache.druid.sql.calcite.schema; import org.apache.calcite.schema.Table; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.sql.calcite.table.DatasourceTable; import javax.inject.Inject; - import java.util.Set; public class DruidSchema extends AbstractTableSchema diff --git a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java index cad98cd8409e..9b28df2c0f1b 100644 --- a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java @@ -40,6 +40,7 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -58,7 +59,6 @@ import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.http.SqlQuery; import org.easymock.EasyMock; import org.hamcrest.MatcherAssert; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index bcc2dddb4c9e..90e75e10f9ca 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -63,6 +63,7 @@ import org.apache.druid.server.QuerySchedulerProvider; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.RequestLogLine; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.log.RequestLogger; import org.apache.druid.server.log.TestRequestLogger; @@ -88,7 +89,6 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.guice.SqlModule; import org.eclipse.jetty.server.Server; import org.joda.time.DateTime; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java index 1df137e772c3..c9dd67b5c7da 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java @@ -32,6 +32,7 @@ import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; @@ -47,7 +48,6 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index ecd104d0b669..0afb275a4509 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -81,6 +81,7 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.ForbiddenException; @@ -97,7 +98,6 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder; import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index d366845bb9b2..e11c12313734 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -68,8 +68,8 @@ import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.virtual.NestedFieldVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; -import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java index a439f1d19ea4..fc9c63cc6675 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java @@ -37,6 +37,7 @@ import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.planner.CalciteRulesManager; @@ -48,7 +49,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index 974edbd3a588..2ac84ed76de1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -26,15 +26,14 @@ import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; -import org.apache.druid.segment.metadata.TestTimelineServerView; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java index dc670af203e4..4a8a7e33c24d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -89,7 +89,8 @@ public Optional build( } @Test - public void testBuild() { + public void testBuild() + { segmentDataSourceNames.add("foo"); joinableDataSourceNames.add("foo"); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index ed7f083dda89..4f0aaaa8e9df 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -88,7 +88,6 @@ import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.coordinator.BytesAccumulatingResponseHandler; -import org.apache.druid.server.coordinator.simulate.TestServerInventoryView; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index dc716b2a45c1..14e1912a93a9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -43,10 +43,10 @@ import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.metadata.TestTimelineServerView; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.log.NoopRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.AuthConfig; @@ -57,7 +57,6 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; @@ -79,7 +78,6 @@ import org.easymock.EasyMock; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 7e012b435811..d06ad3c484d6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -40,10 +40,10 @@ import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.segment.join.JoinableFactoryWrapper; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.sql.SqlStatementFactory; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java index edfa2fc2d334..854df66aa12e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java @@ -61,10 +61,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.join.table.IndexedTableJoinable; import org.apache.druid.segment.join.table.RowBasedIndexedTable; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index eefbe389e62d..4a3f74b77bdd 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -70,6 +70,7 @@ import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.ResponseContextConfig; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.mocks.MockHttpServletRequest; @@ -103,7 +104,6 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.After; From f18c060e152955eb6e26f600c32fc594fd792d2b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 16:07:56 +0530 Subject: [PATCH 18/82] Fix intellij inspections --- .../java/org/apache/druid/sql/calcite/schema/SystemSchema.java | 2 +- .../calcite/schema/PhysicalDataSourceMetadataBuilderTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 48e3a4a44268..3e134d2e74fa 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -305,7 +305,7 @@ public Enumerable scan(DataContext root) final DataSegment segment = val.getDataSegment(); segmentsAlreadySeen.add(segment.getId()); final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(segment.getId()); - long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; + long numReplicas = 0L, numRows = 0L, isRealtime, isAvailable = 0L; if (partialSegmentData != null) { numReplicas = partialSegmentData.getNumReplicas(); numRows = partialSegmentData.getNumRows(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java index 4a8a7e33c24d..987a6d1e0100 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -52,7 +52,7 @@ public class PhysicalDataSourceMetadataBuilderTest private PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; @Before - public void setUp() throws Exception + public void setUp() { segmentDataSourceNames = Sets.newConcurrentHashSet(); joinableDataSourceNames = Sets.newConcurrentHashSet(); From 03383e6e06056211f237ea6cc4b631ca4954078e Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 16:08:13 +0530 Subject: [PATCH 19/82] Fix QueryableCoordinatorServerView test --- .../druid/client/QueryableCoordinatorServerViewTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java index 0bb391165feb..fccbd1651df8 100644 --- a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java @@ -142,6 +142,7 @@ public void testSingleServerAddedRemovedSegment() throws Exception unannounceSegmentForServer(druidServer, segment); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() @@ -248,6 +249,8 @@ public DataSegment apply(Pair input) } Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() From e5c4b39f43d27b1af831939041a0659e74163d63 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 17:52:26 +0530 Subject: [PATCH 20/82] Complete tests for SMC to verify DataSourceInformation --- .../metadata/SegmentMetadataCacheTest.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 1237aced8254..f300316e6ad8 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -372,6 +372,27 @@ public void testGetTableMapFoo() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo"); + final RowSignature fooRowSignature = fooDs.getRowSignature(); + List columnNames = fooRowSignature.getColumnNames(); + Assert.assertEquals(6, columnNames.size()); + + Assert.assertEquals("__time", columnNames.get(0)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(0)).get()); + + Assert.assertEquals("dim2", columnNames.get(1)); + Assert.assertEquals(ColumnType.STRING, fooRowSignature.getColumnType(columnNames.get(1)).get()); + + Assert.assertEquals("m1", columnNames.get(2)); + Assert.assertEquals(ColumnType.DOUBLE, fooRowSignature.getColumnType(columnNames.get(2)).get()); + + Assert.assertEquals("dim1", columnNames.get(3)); + Assert.assertEquals(ColumnType.STRING, fooRowSignature.getColumnType(columnNames.get(3)).get()); + + Assert.assertEquals("cnt", columnNames.get(4)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(4)).get()); + + Assert.assertEquals("unique_dim1", columnNames.get(5)); + Assert.assertEquals(ColumnType.ofComplex("hyperUnique"), fooRowSignature.getColumnType(columnNames.get(5)).get()); } @Test @@ -379,6 +400,18 @@ public void testGetTableMapFoo2() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo2"); + final RowSignature fooRowSignature = fooDs.getRowSignature(); + List columnNames = fooRowSignature.getColumnNames(); + Assert.assertEquals(3, columnNames.size()); + + Assert.assertEquals("__time", columnNames.get(0)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(0)).get()); + + Assert.assertEquals("dim2", columnNames.get(1)); + Assert.assertEquals(ColumnType.STRING, fooRowSignature.getColumnType(columnNames.get(1)).get()); + + Assert.assertEquals("m1", columnNames.get(2)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(2)).get()); } @Test @@ -396,7 +429,36 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli } ); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); + final RowSignature fooRowSignature = fooDs.getRowSignature(); + List columnNames = fooRowSignature.getColumnNames(); + Assert.assertEquals(9, columnNames.size()); + + Assert.assertEquals("__time", columnNames.get(0)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(0)).get()); + + Assert.assertEquals("numbery", columnNames.get(1)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(1)).get()); + + Assert.assertEquals("numberyArrays", columnNames.get(2)); + Assert.assertEquals(ColumnType.DOUBLE_ARRAY, fooRowSignature.getColumnType(columnNames.get(2)).get()); + + Assert.assertEquals("stringy", columnNames.get(3)); + Assert.assertEquals(ColumnType.STRING, fooRowSignature.getColumnType(columnNames.get(3)).get()); + + Assert.assertEquals("array", columnNames.get(4)); + Assert.assertEquals(ColumnType.LONG_ARRAY, fooRowSignature.getColumnType(columnNames.get(4)).get()); + Assert.assertEquals("nested", columnNames.get(5)); + Assert.assertEquals(ColumnType.ofComplex("json"), fooRowSignature.getColumnType(columnNames.get(5)).get()); + + Assert.assertEquals("cnt", columnNames.get(6)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(6)).get()); + + Assert.assertEquals("m1", columnNames.get(7)); + Assert.assertEquals(ColumnType.DOUBLE, fooRowSignature.getColumnType(columnNames.get(7)).get()); + + Assert.assertEquals("unique_dim1", columnNames.get(8)); + Assert.assertEquals(ColumnType.ofComplex("hyperUnique"), fooRowSignature.getColumnType(columnNames.get(8)).get()); } @Test @@ -406,6 +468,37 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt // least restrictive blend across all segments SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); + + final RowSignature fooRowSignature = fooDs.getRowSignature(); + List columnNames = fooRowSignature.getColumnNames(); + Assert.assertEquals(9, columnNames.size()); + + Assert.assertEquals("__time", columnNames.get(0)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(0)).get()); + + Assert.assertEquals("numbery", columnNames.get(1)); + Assert.assertEquals(ColumnType.DOUBLE, fooRowSignature.getColumnType(columnNames.get(1)).get()); + + Assert.assertEquals("numberyArrays", columnNames.get(2)); + Assert.assertEquals(ColumnType.DOUBLE_ARRAY, fooRowSignature.getColumnType(columnNames.get(2)).get()); + + Assert.assertEquals("stringy", columnNames.get(3)); + Assert.assertEquals(ColumnType.STRING_ARRAY, fooRowSignature.getColumnType(columnNames.get(3)).get()); + + Assert.assertEquals("array", columnNames.get(4)); + Assert.assertEquals(ColumnType.DOUBLE_ARRAY, fooRowSignature.getColumnType(columnNames.get(4)).get()); + + Assert.assertEquals("nested", columnNames.get(5)); + Assert.assertEquals(ColumnType.ofComplex("json"), fooRowSignature.getColumnType(columnNames.get(5)).get()); + + Assert.assertEquals("cnt", columnNames.get(6)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(6)).get()); + + Assert.assertEquals("m1", columnNames.get(7)); + Assert.assertEquals(ColumnType.DOUBLE, fooRowSignature.getColumnType(columnNames.get(7)).get()); + + Assert.assertEquals("unique_dim1", columnNames.get(8)); + Assert.assertEquals(ColumnType.ofComplex("hyperUnique"), fooRowSignature.getColumnType(columnNames.get(8)).get()); } /** From 45378d53821a2899c2e56b9d503c1426fd786cad Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 17:52:41 +0530 Subject: [PATCH 21/82] Add comments --- .../QueryableCoordinatorServerView.java | 36 ++++++++++++++----- .../druid/client/selector/ServerSelector.java | 6 ++++ .../metadata/DataSourceInformation.java | 2 +- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index b5f8973519f9..719562a73716 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -43,6 +43,12 @@ /** * ServerView of coordinator for the state of segments being loaded in the cluster. + *
    + * This class simply extends {@link BrokerServerView} and implements methods from {@link CoordinatorTimeline} + * for backward compatibility. This newer implementation is primarily required by + * {@link org.apache.druid.segment.metadata.SegmentMetadataCache} which will run on the Coordinator. + *
    + * Once this class is stable {@link CoordinatorServerView} should be removed. */ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline @@ -71,20 +77,34 @@ public boolean isAwaitInitializationOnStart() this.baseView = baseView; } + /** + * Internally this class maintains a timeline of {@link ServerSelector}. + * This method returns a newline of the object {@link SegmentLoadInfo}. + * + * @param dataSource dataSoruce + * @return timeline for the given dataSource + */ @Override public VersionedIntervalTimeline getTimeline(DataSource dataSource) { String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (lock) { - // build a new timeline? - VersionedIntervalTimeline timeline = timelines.get(table); - Collection x = timeline.iterateAllObjects(); - VersionedIntervalTimeline newTimeline = new VersionedIntervalTimeline<>(Comparator.naturalOrder()); - newTimeline.addAll(x.stream().map(v -> new VersionedIntervalTimeline.PartitionChunkEntry( - v.getSegment().getInterval(), v.getSegment().getVersion(), v.getSegment().getShardSpec().createChunk(v.toSegmentLoadInfo()))).iterator()); + VersionedIntervalTimeline timeline; - return newTimeline; + synchronized (lock) { + timeline = timelines.get(table); } + + VersionedIntervalTimeline newTimeline = + new VersionedIntervalTimeline<>(Comparator.naturalOrder()); + newTimeline.addAll( + timeline.iterateAllObjects().stream() + .map(serverSelector -> new VersionedIntervalTimeline.PartitionChunkEntry<>( + serverSelector.getSegment().getInterval(), + serverSelector.getSegment().getVersion(), + serverSelector.getSegment().getShardSpec().createChunk(serverSelector.toSegmentLoadInfo()) + )).iterator()); + + return newTimeline; } @Override diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 6879600878e2..0050edfa8f81 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -217,6 +217,12 @@ public boolean hasData() return segment.get().hasData(); } + /** + * This conversion is required to make the newer {@link org.apache.druid.client.QueryableCoordinatorServerView} + * implement methods from {@link org.apache.druid.client.CoordinatorTimeline} + * + * @return {@link SegmentLoadInfo} + */ public SegmentLoadInfo toSegmentLoadInfo() { List allServers = getAllServers(); diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index 2e3257e100c1..b8e8c7d5389c 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -26,7 +26,7 @@ import java.util.Objects; /** - * Encapsulates schema information of a dataSource. + * Encapsulates information of a dataSource like schema. */ public class DataSourceInformation { From 2327b12b41382a2b41c20f188cfd55b3509ce9c3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 00:25:03 +0530 Subject: [PATCH 22/82] Refactor SegmentMetadataCacheTest and BrokerSegmentMetadataCacheTest --- .../SegmentDataCacheConcurrencyTest.java | 6 +- .../metadata/SegmentMetadataCacheCommon.java | 355 ++++++- .../metadata/SegmentMetadataCacheTest.java | 512 +-------- .../BrokerSegmentMetadataCacheTest.java | 990 ++---------------- 4 files changed, 444 insertions(+), 1419 deletions(-) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 1746d36b3e16..6a333a9dde02 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -85,9 +85,8 @@ public class SegmentDataCacheConcurrencyTest extends SegmentMetadataCacheCommon { private static final String DATASOURCE = "datasource"; - + static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); private File tmpDir; - private SpecificSegmentsQuerySegmentWalker walker; private TestServerInventoryView inventoryView; private BrokerServerView serverView; private SegmentMetadataCache schema; @@ -96,6 +95,7 @@ public class SegmentDataCacheConcurrencyTest extends SegmentMetadataCacheCommon @Before public void setUp() throws Exception { + setUpCommon(); tmpDir = temporaryFolder.newFolder(); walker = new SpecificSegmentsQuerySegmentWalker(conglomerate); inventoryView = new TestServerInventoryView(); @@ -106,8 +106,10 @@ public void setUp() throws Exception } @After + @Override public void tearDown() throws Exception { + super.tearDown(); exec.shutdownNow(); walker.close(); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 897f9f0a6463..39bec4cdebf8 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -22,33 +22,59 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.MapInputRowParser; import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; +import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.junit.Assert; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import java.io.File; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public abstract class SegmentMetadataCacheCommon { @@ -69,8 +95,6 @@ public abstract class SegmentMetadataCacheCommon null ); - static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - final List ROWS1 = ImmutableList.of( createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), @@ -83,15 +107,17 @@ public abstract class SegmentMetadataCacheCommon createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) ); - static QueryRunnerFactoryConglomerate conglomerate; - static Closer resourceCloser; - static QueryToolChestWarehouse queryToolChestWarehouse; + public QueryRunnerFactoryConglomerate conglomerate; + public Closer resourceCloser; + public QueryToolChestWarehouse queryToolChestWarehouse; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + public SpecificSegmentsQuerySegmentWalker walker; + public TestTimelineServerView serverView; + public List druidServers; - @BeforeClass - public static void setUpClass() + public void setUpCommon() { resourceCloser = Closer.create(); conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); @@ -105,8 +131,174 @@ public > QueryToolChest getToolChest }; } - @AfterClass - public static void tearDownClass() throws IOException + public void setupData() throws Exception + { + final File tmpDir = temporaryFolder.newFolder(); + final QueryableIndex index1 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(ROWS1) + .buildMMappedIndex(); + + final QueryableIndex index2 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "2")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new LongSumAggregatorFactory("m1", "m1")) + .withRollup(false) + .build() + ) + .rows(ROWS2) + .buildMMappedIndex(); + + final InputRowSchema rowSchema = new InputRowSchema( + new TimestampSpec("t", null, null), + DimensionsSpec.builder().useSchemaDiscovery(true).build(), + null + ); + final List autoRows1 = ImmutableList.of( + createRow( + ImmutableMap.builder() + .put("t", "2023-01-01T00:00Z") + .put("numbery", 1.1f) + .put("numberyArrays", ImmutableList.of(1L, 2L, 3L)) + .put("stringy", ImmutableList.of("a", "b", "c")) + .put("array", ImmutableList.of(1.1, 2.2, 3.3)) + .put("nested", ImmutableMap.of("x", 1L, "y", 2L)) + .build(), + rowSchema + ) + ); + final List autoRows2 = ImmutableList.of( + createRow( + ImmutableMap.builder() + .put("t", "2023-01-02T00:00Z") + .put("numbery", 1L) + .put("numberyArrays", ImmutableList.of(3.3, 2.2, 3.1)) + .put("stringy", "a") + .put("array", ImmutableList.of(1L, 2L, 3L)) + .put("nested", "hello") + .build(), + rowSchema + ) + ); + + final QueryableIndex indexAuto1 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec(rowSchema.getTimestampSpec()) + .withDimensionsSpec(rowSchema.getDimensionsSpec()) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows1) + .buildMMappedIndex(); + + final QueryableIndex indexAuto2 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec( + new TimestampSpec("t", null, null) + ) + .withDimensionsSpec( + DimensionsSpec.builder().useSchemaDiscovery(true).build() + ) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows2) + .buildMMappedIndex(); + + walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( + DataSegment.builder() + .dataSource(DATASOURCE1) + .interval(Intervals.of("2000/P1Y")) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index1 + ).add( + DataSegment.builder() + .dataSource(DATASOURCE1) + .interval(Intervals.of("2001/P1Y")) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index2 + ).add( + DataSegment.builder() + .dataSource(DATASOURCE2) + .interval(index2.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index2 + ).add( + DataSegment.builder() + .dataSource(SOME_DATASOURCE) + .interval(Intervals.of("2023-01-01T00Z/P1D")) + .version("1") + .shardSpec(new LinearShardSpec(1)) + .size(0) + .build(), + indexAuto1 + ).add( + DataSegment.builder() + .dataSource(SOME_DATASOURCE) + .interval(Intervals.of("2023-01-02T00Z/P1D")) + .version("1") + .shardSpec(new LinearShardSpec(1)) + .size(0) + .build(), + indexAuto2 + ); + final DataSegment segment1 = new DataSegment( + "foo3", + Intervals.of("2012/2013"), + "version3", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(2, 3), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + final List realtimeSegments = ImmutableList.of(segment1); + serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); + druidServers = serverView.getDruidServers(); + } + + public void tearDown() throws Exception { resourceCloser.close(); } @@ -134,4 +326,145 @@ QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of())) ); } + + public void checkStaleDatasourceRefresh(SegmentMetadataCache schema) throws IOException + { + Set segments = new HashSet<>(); + Set datasources = new HashSet<>(); + datasources.add("wat"); + Assert.assertNull(schema.getDatasource("wat")); + schema.refresh(segments, datasources); + Assert.assertNull(schema.getDatasource("wat")); + } + + public void checkRefreshShouldEmitMetrics( + SegmentMetadataCache schema, + String dataSource, + StubServiceEmitter emitter, + CountDownLatch addSegmentLatch + ) + throws IOException, InterruptedException + { + List segments = ImmutableList.of( + newSegment(dataSource, 1), + newSegment(dataSource, 2) + ); + serverView.addSegment(segments.get(0), ServerType.HISTORICAL); + serverView.addSegment(segments.get(1), ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(dataSource)); + + emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); + emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); + } + + public void checkNullAvailableSegmentMetadata(SegmentMetadataCache schema) throws IOException + { + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // remove one of the segments with datasource "foo" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + } + + public void checkNullDatasource(SegmentMetadataCache schema) throws IOException + { + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // segments contains two segments with datasource "foo" and one with datasource "foo2" + // let's remove the only segment with datasource "foo2" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + } + + public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) throws InterruptedException + { + Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); + final List segments = segmentsMetadata.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // find the only segment with datasource "foo2" + final DataSegment existingSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(existingSegment); + final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); + // update AvailableSegmentMetadata of existingSegment with numRows=5 + AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); + schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); + // find a druidServer holding existingSegment + final Pair pair = druidServers + .stream() + .flatMap(druidServer -> druidServer + .iterateAllSegments() + .stream() + .filter(segment -> segment.getId().equals(existingSegment.getId())) + .map(segment -> Pair.of(druidServer, segment)) + ) + .findAny() + .orElse(null); + Assert.assertNotNull(pair); + final ImmutableDruidServer server = pair.lhs; + Assert.assertNotNull(server); + final DruidServerMetadata druidServerMetadata = server.getMetadata(); + // invoke SegmentMetadataCache#addSegment on existingSegment + schema.addSegment(druidServerMetadata, existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + // get the only segment with datasource "foo2" + final DataSegment currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); + Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); + // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before + Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); + } + + public DataSegment newSegment(String datasource, int partitionId) + { + return new DataSegment( + datasource, + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(partitionId, 0), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + } } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index f300316e6ad8..f016aa0c93a3 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -20,7 +20,6 @@ package org.apache.druid.segment.metadata; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -28,37 +27,22 @@ import com.google.common.collect.Sets; import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.aggregation.CountAggregatorFactory; -import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; -import org.apache.druid.segment.IndexBuilder; -import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; -import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryResponse; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -66,20 +50,15 @@ import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.DataSegment.PruneSpecsHolder; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.partition.LinearShardSpec; -import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.util.EnumSet; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -91,188 +70,33 @@ public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon { // Timeout to allow (rapid) debugging, while not blocking tests with errors. - private static final int WAIT_TIMEOUT_SECS = 6; - - private SpecificSegmentsQuerySegmentWalker walker; - private TestTimelineServerView serverView; - private List druidServers; + private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); + static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); private SegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); - private CountDownLatch refreshLatch = new CountDownLatch(1); - private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); @Before - public void setUp() throws Exception + public void setup() throws Exception { - final File tmpDir = temporaryFolder.newFolder(); - final QueryableIndex index1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(ROWS1) - .buildMMappedIndex(); - - final QueryableIndex index2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "2")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics(new LongSumAggregatorFactory("m1", "m1")) - .withRollup(false) - .build() - ) - .rows(ROWS2) - .buildMMappedIndex(); - - final InputRowSchema rowSchema = new InputRowSchema( - new TimestampSpec("t", null, null), - DimensionsSpec.builder().useSchemaDiscovery(true).build(), - null - ); - final List autoRows1 = ImmutableList.of( - createRow( - ImmutableMap.builder() - .put("t", "2023-01-01T00:00Z") - .put("numbery", 1.1f) - .put("numberyArrays", ImmutableList.of(1L, 2L, 3L)) - .put("stringy", ImmutableList.of("a", "b", "c")) - .put("array", ImmutableList.of(1.1, 2.2, 3.3)) - .put("nested", ImmutableMap.of("x", 1L, "y", 2L)) - .build(), - rowSchema - ) - ); - final List autoRows2 = ImmutableList.of( - createRow( - ImmutableMap.builder() - .put("t", "2023-01-02T00:00Z") - .put("numbery", 1L) - .put("numberyArrays", ImmutableList.of(3.3, 2.2, 3.1)) - .put("stringy", "a") - .put("array", ImmutableList.of(1L, 2L, 3L)) - .put("nested", "hello") - .build(), - rowSchema - ) - ); + setUpCommon(); + setupData(); + } - final QueryableIndex indexAuto1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec(rowSchema.getTimestampSpec()) - .withDimensionsSpec(rowSchema.getDimensionsSpec()) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows1) - .buildMMappedIndex(); - - final QueryableIndex indexAuto2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec( - new TimestampSpec("t", null, null) - ) - .withDimensionsSpec( - DimensionsSpec.builder().useSchemaDiscovery(true).build() - ) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows2) - .buildMMappedIndex(); - - walker = new SpecificSegmentsQuerySegmentWalker(SegmentMetadataCacheCommon.conglomerate).add( - DataSegment.builder() - .dataSource(DATASOURCE1) - .interval(Intervals.of("2000/P1Y")) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index1 - ).add( - DataSegment.builder() - .dataSource(DATASOURCE1) - .interval(Intervals.of("2001/P1Y")) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index2 - ).add( - DataSegment.builder() - .dataSource(DATASOURCE2) - .interval(index2.getDataInterval()) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index2 - ).add( - DataSegment.builder() - .dataSource(SOME_DATASOURCE) - .interval(Intervals.of("2023-01-01T00Z/P1D")) - .version("1") - .shardSpec(new LinearShardSpec(1)) - .size(0) - .build(), - indexAuto1 - ).add( - DataSegment.builder() - .dataSource(SOME_DATASOURCE) - .interval(Intervals.of("2023-01-02T00Z/P1D")) - .version("1") - .shardSpec(new LinearShardSpec(1)) - .size(0) - .build(), - indexAuto2 - ); - final DataSegment segment1 = new DataSegment( - "foo3", - Intervals.of("2012/2013"), - "version3", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(2, 3), - null, - 1, - 100L, - PruneSpecsHolder.DEFAULT - ); - final List realtimeSegments = ImmutableList.of(segment1); - serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); - druidServers = serverView.getDruidServers(); + @After + @Override + public void tearDown() throws Exception + { + super.tearDown(); + if (runningSchema != null) { + runningSchema.stop(); + } + walker.close(); } public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { - return buildSchemaMarkAndTableLatch(SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT); + return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); } public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException @@ -308,48 +132,6 @@ public void markDataSourceAsNeedRebuild(String datasource) return runningSchema; } - public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedException - { - Preconditions.checkState(runningSchema == null); - runningSchema = new SegmentMetadataCache( - getQueryLifecycleFactory(walker), - serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter() - ) - { - @Override - public void markDataSourceAsNeedRebuild(String datasource) - { - super.markDataSourceAsNeedRebuild(datasource); - markDataSourceLatch.countDown(); - } - - @Override - @VisibleForTesting - public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException - { - super.refresh(segmentsToRefresh, dataSourcesToRebuild); - refreshLatch.countDown(); - } - }; - - runningSchema.start(); - runningSchema.awaitInitialization(); - return runningSchema; - } - - @After - public void tearDown() throws Exception - { - if (runningSchema != null) { - runningSchema.stop(); - } - walker.close(); - } - @Test public void testGetTableMap() throws InterruptedException { @@ -363,8 +145,8 @@ public void testGetTableMap() throws InterruptedException @Test public void testSchemaInit() throws InterruptedException { - SegmentMetadataCache schema2 = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema2.getDatasourceNames()); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); } @Test @@ -511,99 +293,21 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt public void testAvailableSegmentMetadataNumRows() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); - final List segments = segmentsMetadata.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // find the only segment with datasource "foo2" - final DataSegment existingSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(existingSegment); - final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); - // update AvailableSegmentMetadata of existingSegment with numRows=5 - AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); - schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); - // find a druidServer holding existingSegment - final Pair pair = druidServers - .stream() - .flatMap(druidServer -> druidServer - .iterateAllSegments() - .stream() - .filter(segment -> segment.getId().equals(existingSegment.getId())) - .map(segment -> Pair.of(druidServer, segment)) - ) - .findAny() - .orElse(null); - Assert.assertNotNull(pair); - final ImmutableDruidServer server = pair.lhs; - Assert.assertNotNull(server); - final DruidServerMetadata druidServerMetadata = server.getMetadata(); - // invoke SegmentMetadataCache#addSegment on existingSegment - schema.addSegment(druidServerMetadata, existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - // get the only segment with datasource "foo2" - final DataSegment currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); - Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); - // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before - Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); + checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // segments contains two segments with datasource "foo" and one with datasource "foo2" - // let's remove the only segment with datasource "foo2" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // remove one of the segments with datasource "foo" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + checkNullAvailableSegmentMetadata(schema); } @Test @@ -669,7 +373,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -710,7 +414,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -755,7 +459,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -797,7 +501,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -836,7 +540,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -892,7 +596,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -951,7 +655,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -984,7 +688,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1030,7 +734,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1077,118 +781,6 @@ public void removeServerSegment(final DruidServerMetadata server, final DataSegm Assert.assertEquals(0, metadata.getNumReplicas()); // brokers are not counted as replicas yet } - /** - * Test actions on the cache. The current design of the cache makes testing far harder - * than it should be. - * - * - The cache is refreshed on a schedule. - * - Datasources are added to the refresh queue via an unsynchronized thread. - * - The refresh loop always refreshes since one of the segments is dynamic. - * - * The use of latches tries to keep things synchronized, but there are many - * moving parts. A simpler technique is sorely needed. - */ - @Test - public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws InterruptedException - { - SegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DataSourceInformation fooTable = schema3.getDatasource("foo"); - Assert.assertNotNull(fooTable); - - markDataSourceLatch = new CountDownLatch(1); - refreshLatch = new CountDownLatch(1); - final DataSegment someNewBrokerSegment = new DataSegment( - "foo", - Intervals.of("2012/2013"), - "version1", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(2, 3), - null, - 1, - 100L, - PruneSpecsHolder.DEFAULT - ); - serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); - Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for build twice - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) - refreshLatch = new CountDownLatch(1); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - - fooTable = schema3.getDatasource("foo"); - Assert.assertNotNull(fooTable); - - // now remove it - markDataSourceLatch = new CountDownLatch(1); - refreshLatch = new CountDownLatch(1); - serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); - - Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for build twice - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) - refreshLatch = new CountDownLatch(1); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - - fooTable = schema3.getDatasource("foo"); - Assert.assertNotNull(fooTable); - } - - @Test - public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throws InterruptedException - { - SegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DataSourceInformation fooTable = schema.getDatasource("foo"); - Assert.assertNotNull(fooTable); - - markDataSourceLatch = new CountDownLatch(1); - refreshLatch = new CountDownLatch(1); - final DataSegment someNewBrokerSegment = new DataSegment( - "foo", - Intervals.of("2012/2013"), - "version1", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(2, 3), - null, - 1, - 100L, - PruneSpecsHolder.DEFAULT - ); - serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); - - Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for build twice - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) - refreshLatch = new CountDownLatch(1); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - - fooTable = schema.getDatasource("foo"); - Assert.assertNotNull(fooTable); - - // now remove it - markDataSourceLatch = new CountDownLatch(1); - refreshLatch = new CountDownLatch(1); - serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); - - Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for build twice - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) - refreshLatch = new CountDownLatch(1); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - - fooTable = schema.getDatasource("foo"); - Assert.assertNotNull(fooTable); - } - /** * Ensure that the BrokerInternalQueryConfig context is honored for this internally generated SegmentMetadata Query */ @@ -1235,7 +827,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception SegmentMetadataCache mySchema = new SegmentMetadataCache( factoryMock, serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), internalQueryConfig, new NoopServiceEmitter() @@ -1351,24 +943,19 @@ public void testSegmentMetadataFallbackType() public void testStaleDatasourceRefresh() throws IOException, InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Set segments = new HashSet<>(); - Set datasources = new HashSet<>(); - datasources.add("wat"); - Assert.assertNull(schema.getDatasource("wat")); - schema.refresh(segments, datasources); - Assert.assertNull(schema.getDatasource("wat")); + checkStaleDatasourceRefresh(schema); } @Test public void testRefreshShouldEmitMetrics() throws InterruptedException, IOException { - String datasource = "xyz"; + String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), emitter @@ -1378,7 +965,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { + if (dataSource.equals(segment.getDataSource())) { addSegmentLatch.countDown(); } } @@ -1390,33 +977,6 @@ public void removeSegment(final DataSegment segment) } }; - List segments = ImmutableList.of( - newSegment(datasource, 1), - newSegment(datasource, 2) - ); - serverView.addSegment(segments.get(0), ServerType.HISTORICAL); - serverView.addSegment(segments.get(1), ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); - - emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); - emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); - } - - private static DataSegment newSegment(String datasource, int partitionId) - { - return new DataSegment( - datasource, - Intervals.of("2012/2013"), - "version1", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(partitionId, 0), - null, - 1, - 100L, - PruneSpecsHolder.DEFAULT - ); + checkRefreshShouldEmitMetrics(schema, dataSource, emitter, addSegmentLatch); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index eeec64ca296f..8aa6a36f8f84 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -30,39 +30,19 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.coordinator.NoopCoordinatorClient; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; -import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DataSource; -import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.aggregation.CountAggregatorFactory; -import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; -import org.apache.druid.query.metadata.metadata.ColumnAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; -import org.apache.druid.segment.IndexBuilder; -import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.TestHelper; -import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.RowSignature; -import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.JoinConditionAnalysis; import org.apache.druid.segment.join.Joinable; import org.apache.druid.segment.join.JoinableFactory; @@ -70,14 +50,11 @@ import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; -import org.apache.druid.segment.metadata.TestTimelineServerView; -import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.segment.metadata.SegmentMetadataCacheCommon; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryResponse; -import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -86,26 +63,18 @@ import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.sql.calcite.table.DruidTable; -import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import java.io.File; import java.io.IOException; import java.util.EnumSet; -import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -116,45 +85,29 @@ // test polling from coordinator and when coordinator doesn't return anything // test the result -public class BrokerSegmentMetadataCacheTest extends CalciteTestBase +public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon { private final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final int WAIT_TIMEOUT_SECS = 6; - private SpecificSegmentsQuerySegmentWalker walker; - private TestTimelineServerView serverView; - private List druidServers; private BrokerSegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); private CountDownLatch refreshLatch = new CountDownLatch(1); - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); SegmentManager segmentManager; Set segmentDataSourceNames; Set joinableDataSourceNames; JoinableFactory globalTableJoinable; CountDownLatch getDatasourcesLatch = new CountDownLatch(1); - private QueryRunnerFactoryConglomerate conglomerate; - private Closer resourceCloser; private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); - static final List ROWS1 = ImmutableList.of( - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) - ); - - static final List ROWS2 = ImmutableList.of( - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) - ); @Before public void setUp() throws Exception { + setUpCommon(); + setupData(); segmentDataSourceNames = Sets.newConcurrentHashSet(); joinableDataSourceNames = Sets.newConcurrentHashSet(); @@ -186,173 +139,17 @@ public Optional build( return Optional.empty(); } }; + } - resourceCloser = Closer.create(); - conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); - - final File tmpDir = temporaryFolder.newFolder(); - final QueryableIndex index1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(ROWS1) - .buildMMappedIndex(); - - final QueryableIndex index2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "2")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics(new LongSumAggregatorFactory("m1", "m1")) - .withRollup(false) - .build() - ) - .rows(ROWS2) - .buildMMappedIndex(); - - final InputRowSchema rowSchema = new InputRowSchema( - new TimestampSpec("t", null, null), - DimensionsSpec.builder().useSchemaDiscovery(true).build(), - null - ); - final List autoRows1 = ImmutableList.of( - TestDataBuilder.createRow( - ImmutableMap.builder() - .put("t", "2023-01-01T00:00Z") - .put("numbery", 1.1f) - .put("numberyArrays", ImmutableList.of(1L, 2L, 3L)) - .put("stringy", ImmutableList.of("a", "b", "c")) - .put("array", ImmutableList.of(1.1, 2.2, 3.3)) - .put("nested", ImmutableMap.of("x", 1L, "y", 2L)) - .build(), - rowSchema - ) - ); - final List autoRows2 = ImmutableList.of( - TestDataBuilder.createRow( - ImmutableMap.builder() - .put("t", "2023-01-02T00:00Z") - .put("numbery", 1L) - .put("numberyArrays", ImmutableList.of(3.3, 2.2, 3.1)) - .put("stringy", "a") - .put("array", ImmutableList.of(1L, 2L, 3L)) - .put("nested", "hello") - .build(), - rowSchema - ) - ); - - final QueryableIndex indexAuto1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec(rowSchema.getTimestampSpec()) - .withDimensionsSpec(rowSchema.getDimensionsSpec()) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows1) - .buildMMappedIndex(); - - final QueryableIndex indexAuto2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec( - new TimestampSpec("t", null, null) - ) - .withDimensionsSpec( - DimensionsSpec.builder().useSchemaDiscovery(true).build() - ) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows2) - .buildMMappedIndex(); - - walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( - DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE1) - .interval(Intervals.of("2000/P1Y")) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index1 - ).add( - DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE1) - .interval(Intervals.of("2001/P1Y")) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index2 - ).add( - DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE2) - .interval(index2.getDataInterval()) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index2 - ).add( - DataSegment.builder() - .dataSource(CalciteTests.SOME_DATASOURCE) - .interval(Intervals.of("2023-01-01T00Z/P1D")) - .version("1") - .shardSpec(new LinearShardSpec(1)) - .size(0) - .build(), - indexAuto1 - ).add( - DataSegment.builder() - .dataSource(CalciteTests.SOME_DATASOURCE) - .interval(Intervals.of("2023-01-02T00Z/P1D")) - .version("1") - .shardSpec(new LinearShardSpec(1)) - .size(0) - .build(), - indexAuto2 - ); - final DataSegment segment1 = new DataSegment( - "foo3", - Intervals.of("2012/2013"), - "version3", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(2, 3), - null, - 1, - 100L, - DataSegment.PruneSpecsHolder.DEFAULT - ); - final List realtimeSegments = ImmutableList.of(segment1); - serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); - druidServers = serverView.getDruidServers(); + @After + @Override + public void tearDown() throws Exception + { + super.tearDown(); + if (runningSchema != null) { + runningSchema.stop(); + } + walker.close(); } public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException @@ -431,16 +228,6 @@ public void refresh(final Set segmentsToRefresh, final Set da return runningSchema; } - @After - public void tearDown() throws Exception - { - if (runningSchema != null) { - runningSchema.stop(); - } - walker.close(); - resourceCloser.close(); - } - @Test public void testGetTableMap() throws InterruptedException { @@ -454,8 +241,8 @@ public void testGetTableMap() throws InterruptedException @Test public void testSchemaInit() throws InterruptedException { - BrokerSegmentMetadataCache schema2 = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema2.getDatasourceNames()); + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); } @Test @@ -614,586 +401,21 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt public void testAvailableSegmentMetadataNumRows() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); - final List segments = segmentsMetadata.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // find the only segment with datasource "foo2" - final DataSegment existingSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(existingSegment); - final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); - // update AvailableSegmentMetadata of existingSegment with numRows=5 - AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); - schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); - // find a druidServer holding existingSegment - final Pair pair = druidServers - .stream() - .flatMap(druidServer -> druidServer - .iterateAllSegments() - .stream() - .filter(segment -> segment.getId().equals(existingSegment.getId())) - .map(segment -> Pair.of(druidServer, segment)) - ) - .findAny() - .orElse(null); - Assert.assertNotNull(pair); - final ImmutableDruidServer server = pair.lhs; - Assert.assertNotNull(server); - final DruidServerMetadata druidServerMetadata = server.getMetadata(); - // invoke SegmentMetadataCache#addSegment on existingSegment - schema.addSegment(druidServerMetadata, existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - // get the only segment with datasource "foo2" - final DataSegment currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); - Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); - // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before - Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); + checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // segments contains two segments with datasource "foo" and one with datasource "foo2" - // let's remove the only segment with datasource "foo2" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // remove one of the segments with datasource "foo" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); - } - - @Test - public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException - { - BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); - final List segments = segmentsMetadata.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - // find the only realtime segment with datasource "foo3" - final DataSegment existingSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo3")) - .findFirst() - .orElse(null); - Assert.assertNotNull(existingSegment); - final AvailableSegmentMetadata metadata = segmentsMetadata.get(existingSegment.getId()); - Assert.assertEquals(1L, metadata.isRealtime()); - // get the historical server - final ImmutableDruidServer historicalServer = druidServers.stream() - .filter(s -> s.getType().equals(ServerType.HISTORICAL)) - .findAny() - .orElse(null); - - Assert.assertNotNull(historicalServer); - final DruidServerMetadata historicalServerMetadata = historicalServer.getMetadata(); - - // add existingSegment to historical - schema.addSegment(historicalServerMetadata, existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - // get the segment with datasource "foo3" - DataSegment currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo3")) - .findFirst() - .orElse(null); - Assert.assertNotNull(currentSegment); - AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(0L, currentMetadata.isRealtime()); - - ImmutableDruidServer realtimeServer = druidServers.stream() - .filter(s -> s.getType().equals(ServerType.REALTIME)) - .findAny() - .orElse(null); - Assert.assertNotNull(realtimeServer); - // drop existingSegment from realtime task - schema.removeServerSegment(realtimeServer.getMetadata(), existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo3")) - .findFirst() - .orElse(null); - Assert.assertNotNull(currentSegment); - currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(0L, currentMetadata.isRealtime()); - } - - @Test - public void testSegmentAddedCallbackAddNewHistoricalSegment() throws InterruptedException - { - String datasource = "newSegmentAddTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - }; - - serverView.addSegment(newSegment(datasource, 1), ServerType.HISTORICAL); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - AvailableSegmentMetadata metadata = metadatas.get(0); - Assert.assertEquals(0, metadata.isRealtime()); - Assert.assertEquals(0, metadata.getNumRows()); - Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); - } - - @Test - public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedException - { - String datasource = "newSegmentAddTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(2); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - }; - - DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.REALTIME); - serverView.addSegment(segment, ServerType.HISTORICAL); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - AvailableSegmentMetadata metadata = metadatas.get(0); - Assert.assertEquals(0, metadata.isRealtime()); // realtime flag is unset when there is any historical - Assert.assertEquals(0, metadata.getNumRows()); - Assert.assertEquals(2, metadata.getNumReplicas()); - Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); - Assert.assertFalse(schema.getMutableSegments().contains(metadata.getSegment().getId())); - } - - @Test - public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedException - { - String datasource = "newSegmentAddTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - }; - - serverView.addSegment(newSegment(datasource, 1), ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - AvailableSegmentMetadata metadata = metadatas.get(0); - Assert.assertEquals(1, metadata.isRealtime()); - Assert.assertEquals(0, metadata.getNumRows()); - Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); - Assert.assertTrue(schema.getMutableSegments().contains(metadata.getSegment().getId())); - } - - @Test - public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedException - { - String datasource = "newSegmentAddTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - }; - - serverView.addSegment(newSegment(datasource, 1), ServerType.BROKER); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(6, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(0, metadatas.size()); - Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); - } - - @Test - public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws InterruptedException, IOException - { - String datasource = "segmentRemoveTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - CountDownLatch removeSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - - @Override - public void removeSegment(final DataSegment segment) - { - super.removeSegment(segment); - if (datasource.equals(segment.getDataSource())) { - removeSegmentLatch.countDown(); - } - } - }; - - DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(Sets.newHashSet(segment.getId()), Sets.newHashSet(datasource)); - - serverView.removeSegment(segment, ServerType.REALTIME); - Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(6, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(0, metadatas.size()); - Assert.assertFalse(schema.getSegmentsNeedingRefresh().contains(segment.getId())); - Assert.assertFalse(schema.getMutableSegments().contains(segment.getId())); - Assert.assertFalse(schema.getDataSourcesNeedingRebuild().contains(datasource)); - Assert.assertFalse(schema.getDatasourceNames().contains(datasource)); - } - - @Test - public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws InterruptedException, IOException - { - String datasource = "segmentRemoveTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(2); - CountDownLatch removeSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - - @Override - public void removeSegment(final DataSegment segment) - { - super.removeSegment(segment); - if (datasource.equals(segment.getDataSource())) { - removeSegmentLatch.countDown(); - } - } - }; - - List segments = ImmutableList.of( - newSegment(datasource, 1), - newSegment(datasource, 2) - ); - serverView.addSegment(segments.get(0), ServerType.REALTIME); - serverView.addSegment(segments.get(1), ServerType.HISTORICAL); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); - - serverView.removeSegment(segments.get(0), ServerType.REALTIME); - Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - Assert.assertFalse(schema.getSegmentsNeedingRefresh().contains(segments.get(0).getId())); - Assert.assertFalse(schema.getMutableSegments().contains(segments.get(0).getId())); - Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); - Assert.assertTrue(schema.getDatasourceNames().contains(datasource)); - } - - @Test - public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws InterruptedException - { - String datasource = "serverSegmentRemoveTest"; - CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.removeServerSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - removeServerSegmentLatch.countDown(); - } - } - }; - - serverView.addSegment(newSegment(datasource, 1), ServerType.BROKER); - - serverView.removeSegment(newSegment(datasource, 1), ServerType.HISTORICAL); - Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(6, schema.getTotalSegments()); - } - - @Test - public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws InterruptedException - { - String datasource = "serverSegmentRemoveTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - - @Override - public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.removeServerSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - removeServerSegmentLatch.countDown(); - } - } - }; - - DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.HISTORICAL); - serverView.addSegment(segment, ServerType.BROKER); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - serverView.removeSegment(segment, ServerType.BROKER); - Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); - } - - @Test - public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws InterruptedException - { - String datasource = "serverSegmentRemoveTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - - @Override - public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.removeServerSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - removeServerSegmentLatch.countDown(); - } - } - }; - - DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.HISTORICAL); - serverView.addSegment(segment, ServerType.BROKER); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - serverView.removeSegment(segment, ServerType.HISTORICAL); - Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - AvailableSegmentMetadata metadata = metadatas.get(0); - Assert.assertEquals(0, metadata.isRealtime()); - Assert.assertEquals(0, metadata.getNumRows()); - Assert.assertEquals(0, metadata.getNumReplicas()); // brokers are not counted as replicas yet + checkNullAvailableSegmentMetadata(schema); } /** @@ -1210,10 +432,14 @@ public void removeServerSegment(final DruidServerMetadata server, final DataSegm @Test public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws InterruptedException { - BrokerSegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); + BrokerSegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertFalse(fooTable.isJoinable()); + Assert.assertFalse(fooTable.isBroadcast()); markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); @@ -1230,6 +456,8 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte 100L, DataSegment.PruneSpecsHolder.DEFAULT ); + segmentDataSourceNames.add("foo"); + joinableDataSourceNames.add("foo"); serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); // wait for build twice @@ -1238,12 +466,18 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertTrue(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertTrue(fooTable.isJoinable()); + Assert.assertTrue(fooTable.isBroadcast()); // now remove it markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); + joinableDataSourceNames.remove("foo"); + segmentDataSourceNames.remove("foo"); serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1253,8 +487,12 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertFalse(fooTable.isJoinable()); + Assert.assertFalse(fooTable.isBroadcast()); } @Test @@ -1264,6 +502,11 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertFalse(fooTable.isJoinable()); + Assert.assertFalse(fooTable.isBroadcast()); markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); @@ -1280,6 +523,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw 100L, DataSegment.PruneSpecsHolder.DEFAULT ); + segmentDataSourceNames.add("foo"); serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1291,10 +535,17 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + // Should not be a GlobalTableDataSource for now, because isGlobal is couple with joinability. Ideally this will be + // changed in the future and we should expect. + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertTrue(fooTable.isBroadcast()); + Assert.assertFalse(fooTable.isJoinable()); // now remove it markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); + segmentDataSourceNames.remove("foo"); serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1306,6 +557,10 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertFalse(fooTable.isBroadcast()); + Assert.assertFalse(fooTable.isJoinable()); } /** @@ -1375,115 +630,17 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception } - @Test - public void testSegmentMetadataColumnType() - { - // Verify order is preserved. - final LinkedHashMap columns = new LinkedHashMap<>(); - columns.put( - "a", - new ColumnAnalysis(ColumnType.STRING, ColumnType.STRING.asTypeString(), false, true, 1234, 26, "a", "z", null) - ); - - columns.put( - "count", - new ColumnAnalysis(ColumnType.LONG, ColumnType.LONG.asTypeString(), false, true, 1234, 26, "a", "z", null) - ); - - columns.put( - "b", - new ColumnAnalysis(ColumnType.DOUBLE, ColumnType.DOUBLE.asTypeString(), false, true, 1234, 26, null, null, null) - ); - - RowSignature signature = SegmentMetadataCache.analysisToRowSignature( - new SegmentAnalysis( - "id", - ImmutableList.of(Intervals.utc(1L, 2L)), - columns, - 1234, - 100, - null, - null, - null, - null - ) - ); - - Assert.assertEquals( - RowSignature.builder() - .add("a", ColumnType.STRING) - .add("count", ColumnType.LONG) - .add("b", ColumnType.DOUBLE) - .build(), - signature - ); - } - - @Test - public void testSegmentMetadataFallbackType() - { - RowSignature signature = SegmentMetadataCache.analysisToRowSignature( - new SegmentAnalysis( - "id", - ImmutableList.of(Intervals.utc(1L, 2L)), - new LinkedHashMap<>( - ImmutableMap.of( - "a", - new ColumnAnalysis( - null, - ColumnType.STRING.asTypeString(), - false, - true, - 1234, - 26, - "a", - "z", - null - ), - "count", - new ColumnAnalysis( - null, - ColumnType.LONG.asTypeString(), - false, - true, - 1234, - 26, - "a", - "z", - null - ) - ) - ), - 1234, - 100, - null, - null, - null, - null - ) - ); - Assert.assertEquals( - RowSignature.builder().add("a", ColumnType.STRING).add("count", ColumnType.LONG).build(), - signature - ); - } - @Test public void testStaleDatasourceRefresh() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Set segments = new HashSet<>(); - Set datasources = new HashSet<>(); - datasources.add("wat"); - Assert.assertNull(schema.getDatasource("wat")); - schema.refresh(segments, datasources); - Assert.assertNull(schema.getDatasource("wat")); + checkStaleDatasourceRefresh(schema); } @Test public void testRefreshShouldEmitMetrics() throws InterruptedException, IOException { - String datasource = "xyz"; + String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( @@ -1501,7 +658,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { + if (dataSource.equals(segment.getDataSource())) { addSegmentLatch.countDown(); } } @@ -1513,33 +670,6 @@ public void removeSegment(final DataSegment segment) } }; - List segments = ImmutableList.of( - newSegment(datasource, 1), - newSegment(datasource, 2) - ); - serverView.addSegment(segments.get(0), ServerType.HISTORICAL); - serverView.addSegment(segments.get(1), ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); - - emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); - emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); - } - - private static DataSegment newSegment(String datasource, int partitionId) - { - return new DataSegment( - datasource, - Intervals.of("2012/2013"), - "version1", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(partitionId, 0), - null, - 1, - 100L, - DataSegment.PruneSpecsHolder.DEFAULT - ); + checkRefreshShouldEmitMetrics(schema, dataSource, emitter, addSegmentLatch); } } From d4ece6a138a297c4c21a7c0905b9dbd4ca367193 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 14:15:58 +0530 Subject: [PATCH 23/82] Test fetching ds schema from coordinator in BrokerSegmentMetadataCacheTest --- .../metadata/SegmentMetadataCache.java | 2 + .../metadata/SegmentMetadataCacheCommon.java | 74 +++++++- .../metadata/SegmentMetadataCacheTest.java | 42 +---- .../schema/BrokerSegmentMetadataCache.java | 7 +- .../BrokerSegmentMetadataCacheTest.java | 162 ++++++++++++------ 5 files changed, 187 insertions(+), 100 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index e5709fac0dbd..6d80bd9390a8 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -61,7 +61,9 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.Types; +import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.QueryResponse; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.security.Access; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 39bec4cdebf8..096e96ed39e7 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -31,43 +31,60 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.Query; +import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.TableDataSource; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; +import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.QueryableIndexStorageAdapter; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.join.table.IndexedTable; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.QueryResponse; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Rule; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -117,6 +134,13 @@ public abstract class SegmentMetadataCacheCommon public TestTimelineServerView serverView; public List druidServers; + public QueryableIndex index1; + public QueryableIndex index2; + + public QueryableIndex indexAuto1; + public QueryableIndex indexAuto2; + public DataSegment realtimeSegment1; + public void setUpCommon() { resourceCloser = Closer.create(); @@ -134,7 +158,7 @@ public > QueryToolChest getToolChest public void setupData() throws Exception { final File tmpDir = temporaryFolder.newFolder(); - final QueryableIndex index1 = IndexBuilder.create() + index1 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( @@ -150,7 +174,7 @@ public void setupData() throws Exception .rows(ROWS1) .buildMMappedIndex(); - final QueryableIndex index2 = IndexBuilder.create() + index2 = IndexBuilder.create() .tmpDir(new File(tmpDir, "2")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( @@ -194,7 +218,7 @@ public void setupData() throws Exception ) ); - final QueryableIndex indexAuto1 = IndexBuilder.create() + indexAuto1 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( @@ -212,7 +236,7 @@ public void setupData() throws Exception .rows(autoRows1) .buildMMappedIndex(); - final QueryableIndex indexAuto2 = IndexBuilder.create() + indexAuto2 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( @@ -280,7 +304,7 @@ public void setupData() throws Exception .build(), indexAuto2 ); - final DataSegment segment1 = new DataSegment( + realtimeSegment1 = new DataSegment( "foo3", Intervals.of("2012/2013"), "version3", @@ -293,7 +317,8 @@ public void setupData() throws Exception 100L, DataSegment.PruneSpecsHolder.DEFAULT ); - final List realtimeSegments = ImmutableList.of(segment1); + + final List realtimeSegments = ImmutableList.of(realtimeSegment1); serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); druidServers = serverView.getDruidServers(); } @@ -451,6 +476,43 @@ public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) th Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); } + public void checkRunSegmentMetadataQueryWithContext(SegmentMetadataCache schema, QueryLifecycleFactory factoryMock, QueryLifecycle lifecycleMock) + { + Map queryContext = ImmutableMap.of( + QueryContexts.PRIORITY_KEY, 5, + QueryContexts.BROKER_PARALLEL_MERGE_KEY, false + ); + + DataSegment segment = newSegment("test", 0); + List segmentIterable = ImmutableList.of(segment.getId()); + + // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. + SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(segment.getDataSource()), + new MultipleSpecificSegmentSpec( + segmentIterable.stream() + .map(SegmentId::toDescriptor).collect(Collectors.toList())), + new AllColumnIncluderator(), + false, + queryContext, + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); + // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context + EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) + .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); + + EasyMock.replay(factoryMock, lifecycleMock); + + schema.runSegmentMetadataQuery(segmentIterable); + + EasyMock.verify(factoryMock, lifecycleMock); + } + public DataSegment newSegment(String datasource, int partitionId) { return new DataSegment( diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index f016aa0c93a3..4c969464a6ab 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -142,13 +142,6 @@ public void testGetTableMap() throws InterruptedException Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), tableNames); } - @Test - public void testSchemaInit() throws InterruptedException - { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); - } - @Test public void testGetTableMapFoo() throws InterruptedException { @@ -787,11 +780,6 @@ public void removeServerSegment(final DruidServerMetadata server, final DataSegm @Test public void testRunSegmentMetadataQueryWithContext() throws Exception { - Map queryContext = ImmutableMap.of( - QueryContexts.PRIORITY_KEY, 5, - QueryContexts.BROKER_PARALLEL_MERGE_KEY, false - ); - String brokerInternalQueryConfigJson = "{\"context\": { \"priority\": 5} }"; TestHelper.makeJsonMapper(); @@ -802,24 +790,6 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception InternalQueryConfig.class ); - DataSegment segment = newSegment("test", 0); - List segmentIterable = ImmutableList.of(segment.getId()); - - // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. - SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( - new TableDataSource(segment.getDataSource()), - new MultipleSpecificSegmentSpec( - segmentIterable.stream() - .map(SegmentId::toDescriptor).collect(Collectors.toList())), - new AllColumnIncluderator(), - false, - queryContext, - EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), - false, - null, - null - ); - QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); @@ -833,17 +803,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception new NoopServiceEmitter() ); - EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); - // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context - EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) - .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); - - EasyMock.replay(factoryMock, lifecycleMock); - - mySchema.runSegmentMetadataQuery(segmentIterable); - - EasyMock.verify(factoryMock, lifecycleMock); - + checkRunSegmentMetadataQueryWithContext(mySchema, factoryMock, lifecycleMock); } @Test diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 8c2369aaab62..921bbcf85461 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -104,11 +105,7 @@ public void refresh(final Set segmentsToRefresh, final Set da tables.putAll(polledDataSourceMetadata); // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. - segmentsToRefresh.forEach(segment -> { - if (polledDataSourceMetadata.containsKey(segment.getDataSource())) { - segmentsToRefresh.remove(segment); - } - }); + segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); // Refresh the remaining segments. final Set refreshed = refreshSegments(segmentsToRefresh); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 8aa6a36f8f84..5480cd61f352 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -26,11 +26,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.coordinator.CoordinatorClient; import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequences; @@ -42,7 +45,9 @@ import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; +import org.apache.druid.segment.QueryableIndexStorageAdapter; import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinConditionAnalysis; import org.apache.druid.segment.join.Joinable; import org.apache.druid.segment.join.JoinableFactory; @@ -74,17 +79,17 @@ import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -// test polling from coordinator and when coordinator doesn't return anything -// test the result public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon { private final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); @@ -154,10 +159,10 @@ public void tearDown() throws Exception public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { - return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); + return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT, new NoopCoordinatorClient()); } - public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMetadataCacheConfig config) throws InterruptedException + public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMetadataCacheConfig config, CoordinatorClient coordinatorClient) throws InterruptedException { Preconditions.checkState(runningSchema == null); runningSchema = new BrokerSegmentMetadataCache( @@ -168,7 +173,7 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMeta new InternalQueryConfig(), new NoopServiceEmitter(), new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() + coordinatorClient ) { @Override @@ -215,8 +220,9 @@ public void markDataSourceAsNeedRebuild(String datasource) @Override @VisibleForTesting - public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws - IOException + public void refresh( + final Set segmentsToRefresh, + final Set dataSourcesToRebuild) throws IOException { super.refresh(segmentsToRefresh, dataSourcesToRebuild); refreshLatch.countDown(); @@ -229,17 +235,109 @@ public void refresh(final Set segmentsToRefresh, final Set da } @Test - public void testGetTableMap() throws InterruptedException + public void testGetAllDsSchemaFromCoordinator() throws InterruptedException { - BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); + final RowSignature dataSource1RowSignature = new QueryableIndexStorageAdapter(index1).getRowSignature(); + final RowSignature dataSource2RowSignature = new QueryableIndexStorageAdapter(index2).getRowSignature(); + final RowSignature someDataSourceRowSignature = new QueryableIndexStorageAdapter(indexAuto1).getRowSignature(); + final RowSignature foo3RowSignature = new QueryableIndexStorageAdapter(indexAuto2).getRowSignature(); + + NoopCoordinatorClient coordinatorClient = new NoopCoordinatorClient() { + @Override + public ListenableFuture> fetchDataSourceInformation(Set datasources) + { + Map dataSourceInformationMap = new HashMap<>(); + dataSourceInformationMap.put(DATASOURCE1, new DataSourceInformation(DATASOURCE1, dataSource1RowSignature)); + dataSourceInformationMap.put(DATASOURCE2, new DataSourceInformation(DATASOURCE2, dataSource2RowSignature)); + dataSourceInformationMap.put(SOME_DATASOURCE, new DataSourceInformation(SOME_DATASOURCE, someDataSourceRowSignature)); + dataSourceInformationMap.put("foo3", new DataSourceInformation("foo3", foo3RowSignature)); + + return Futures.immediateFuture(new ArrayList<>(dataSourceInformationMap.values())); + } + }; + + QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); + + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + factoryMock, + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + coordinatorClient + ); + schema.start(); + schema.awaitInitialization(); final Set tableNames = schema.getDatasourceNames(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), tableNames); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE, "foo3"), tableNames); + + Assert.assertEquals(dataSource1RowSignature, schema.getPhysicalDatasourceMetadata(DATASOURCE1).rowSignature()); + Assert.assertEquals(dataSource2RowSignature, schema.getPhysicalDatasourceMetadata(DATASOURCE2).rowSignature()); + Assert.assertEquals(someDataSourceRowSignature, schema.getPhysicalDatasourceMetadata(SOME_DATASOURCE).rowSignature()); + Assert.assertEquals(foo3RowSignature, schema.getPhysicalDatasourceMetadata("foo3").rowSignature()); + } + + @Test + public void testGetFewDsSchemaFromCoordinator() throws InterruptedException + { + final RowSignature dataSource1RowSignature = new QueryableIndexStorageAdapter(index1).getRowSignature(); + final RowSignature dataSource2RowSignature = new QueryableIndexStorageAdapter(index2).getRowSignature(); + final RowSignature someDataSourceRowSignature = new QueryableIndexStorageAdapter(indexAuto1).getRowSignature(); + + NoopCoordinatorClient coordinatorClient = new NoopCoordinatorClient() { + @Override + public ListenableFuture> fetchDataSourceInformation(Set datasources) + { + Map dataSourceInformationMap = new HashMap<>(); + dataSourceInformationMap.put(DATASOURCE1, new DataSourceInformation(DATASOURCE1, dataSource1RowSignature)); + dataSourceInformationMap.put(DATASOURCE2, new DataSourceInformation(DATASOURCE2, dataSource2RowSignature)); + dataSourceInformationMap.put(SOME_DATASOURCE, new DataSourceInformation(SOME_DATASOURCE, someDataSourceRowSignature)); + return Futures.immediateFuture(new ArrayList<>(dataSourceInformationMap.values())); + } + }; + + SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( + new TableDataSource("foo3"), + new MultipleSpecificSegmentSpec(Collections.singletonList(realtimeSegment1.getId().toDescriptor())), + new AllColumnIncluderator(), + false, + ImmutableMap.of(QueryContexts.BROKER_PARALLEL_MERGE_KEY, false), + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); + QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); + EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); + EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) + .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); + + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + factoryMock, + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + coordinatorClient + ); + + EasyMock.replay(factoryMock, lifecycleMock); + + schema.start(); + schema.awaitInitialization(); + + EasyMock.verify(factoryMock, lifecycleMock); } @Test - public void testSchemaInit() throws InterruptedException + public void testGetTableMap() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); @@ -308,7 +406,8 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli { return new SegmentMetadataCache.FirstTypeMergePolicy(); } - } + }, + new NoopCoordinatorClient() ); final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); final DruidTable table = new DatasourceTable(fooDs); @@ -569,11 +668,6 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw @Test public void testRunSegmentMetadataQueryWithContext() throws Exception { - Map queryContext = ImmutableMap.of( - QueryContexts.PRIORITY_KEY, 5, - QueryContexts.BROKER_PARALLEL_MERGE_KEY, false - ); - String brokerInternalQueryConfigJson = "{\"context\": { \"priority\": 5} }"; TestHelper.makeJsonMapper(); @@ -584,24 +678,6 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception InternalQueryConfig.class ); - DataSegment segment = newSegment("test", 0); - List segmentIterable = ImmutableList.of(segment.getId()); - - // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. - SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( - new TableDataSource(segment.getDataSource()), - new MultipleSpecificSegmentSpec( - segmentIterable.stream() - .map(SegmentId::toDescriptor).collect(Collectors.toList())), - new AllColumnIncluderator(), - false, - queryContext, - EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), - false, - null, - null - ); - QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); @@ -617,17 +693,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception new NoopCoordinatorClient() ); - EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); - // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context - EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) - .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); - - EasyMock.replay(factoryMock, lifecycleMock); - - mySchema.runSegmentMetadataQuery(segmentIterable); - - EasyMock.verify(factoryMock, lifecycleMock); - + checkRunSegmentMetadataQueryWithContext(mySchema, factoryMock, lifecycleMock); } @Test From eb1771ffe0f1d0c81463fa448985913f1fc9b769 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 14:45:35 +0530 Subject: [PATCH 24/82] fix checkstyle issue --- .../metadata/SegmentMetadataCache.java | 2 - .../metadata/SegmentMetadataCacheCommon.java | 125 +++++++++--------- .../metadata/SegmentMetadataCacheTest.java | 10 -- .../schema/BrokerSegmentMetadataCache.java | 1 - 4 files changed, 60 insertions(+), 78 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 6d80bd9390a8..e5709fac0dbd 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -61,9 +61,7 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.Types; -import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.QueryResponse; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.security.Access; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 096e96ed39e7..19540e231fe7 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -53,11 +53,7 @@ import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; -import org.apache.druid.segment.QueryableIndexStorageAdapter; -import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.segment.join.table.IndexedTable; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; @@ -84,7 +80,6 @@ import java.io.File; import java.io.IOException; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -159,32 +154,32 @@ public void setupData() throws Exception { final File tmpDir = temporaryFolder.newFolder(); index1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(ROWS1) - .buildMMappedIndex(); + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(ROWS1) + .buildMMappedIndex(); index2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "2")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics(new LongSumAggregatorFactory("m1", "m1")) - .withRollup(false) - .build() - ) - .rows(ROWS2) - .buildMMappedIndex(); + .tmpDir(new File(tmpDir, "2")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new LongSumAggregatorFactory("m1", "m1")) + .withRollup(false) + .build() + ) + .rows(ROWS2) + .buildMMappedIndex(); final InputRowSchema rowSchema = new InputRowSchema( new TimestampSpec("t", null, null), @@ -219,44 +214,44 @@ public void setupData() throws Exception ); indexAuto1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec(rowSchema.getTimestampSpec()) - .withDimensionsSpec(rowSchema.getDimensionsSpec()) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows1) - .buildMMappedIndex(); + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec(rowSchema.getTimestampSpec()) + .withDimensionsSpec(rowSchema.getDimensionsSpec()) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows1) + .buildMMappedIndex(); indexAuto2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec( - new TimestampSpec("t", null, null) - ) - .withDimensionsSpec( - DimensionsSpec.builder().useSchemaDiscovery(true).build() - ) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows2) - .buildMMappedIndex(); + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec( + new TimestampSpec("t", null, null) + ) + .withDimensionsSpec( + DimensionsSpec.builder().useSchemaDiscovery(true).build() + ) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows2) + .buildMMappedIndex(); walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( DataSegment.builder() diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 4c969464a6ab..c60d08f80bc6 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -28,26 +28,17 @@ import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; -import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.QueryResponse; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; -import org.apache.druid.server.security.Access; -import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; @@ -58,7 +49,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 921bbcf85461..f59358586cde 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; From 1440dacec90f31546d04a44bc326469cc84df464 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 17:57:49 +0530 Subject: [PATCH 25/82] Add test for QueryableCoordinatorServerView --- .../client/CoordinatorServerViewTest.java | 373 ------------------ ...Test.java => CoordinatorTimelineTest.java} | 132 +++++-- 2 files changed, 90 insertions(+), 415 deletions(-) delete mode 100644 server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java rename server/src/test/java/org/apache/druid/client/{QueryableCoordinatorServerViewTest.java => CoordinatorTimelineTest.java} (81%) diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java deleted file mode 100644 index 1bca2a0690af..000000000000 --- a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.client; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Function; -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import org.apache.curator.utils.ZKPaths; -import org.apache.druid.curator.CuratorTestBase; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.segment.TestHelper; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; -import org.apache.druid.server.initialization.ZkPathsConfig; -import org.apache.druid.server.metrics.NoopServiceEmitter; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.TimelineLookup; -import org.apache.druid.timeline.TimelineObjectHolder; -import org.apache.druid.timeline.partition.NoneShardSpec; -import org.apache.druid.timeline.partition.PartitionHolder; -import org.joda.time.Interval; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; - -public class CoordinatorServerViewTest extends CuratorTestBase -{ - private final ObjectMapper jsonMapper; - private final ZkPathsConfig zkPathsConfig; - private final String inventoryPath; - - private CountDownLatch segmentViewInitLatch; - private CountDownLatch segmentAddedLatch; - private CountDownLatch segmentRemovedLatch; - - private BatchServerInventoryView baseView; - private CoordinatorServerView overlordServerView; - - public CoordinatorServerViewTest() - { - jsonMapper = TestHelper.makeJsonMapper(); - zkPathsConfig = new ZkPathsConfig(); - inventoryPath = zkPathsConfig.getLiveSegmentsPath(); - } - - @Before - public void setUp() throws Exception - { - setupServerAndCurator(); - curator.start(); - curator.blockUntilConnected(); - } - - @Test - public void testSingleServerAddedRemovedSegment() throws Exception - { - segmentViewInitLatch = new CountDownLatch(1); - segmentAddedLatch = new CountDownLatch(1); - segmentRemovedLatch = new CountDownLatch(1); - - setupViews(); - - final DruidServer druidServer = new DruidServer( - "localhost:1234", - "localhost:1234", - null, - 10000000L, - ServerType.HISTORICAL, - "default_tier", - 0 - ); - - setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper); - - final DataSegment segment = dataSegmentWithIntervalAndVersion("2014-10-20T00:00:00Z/P1D", "v1"); - final int partition = segment.getShardSpec().getPartitionNum(); - final Interval intervals = Intervals.of("2014-10-20T00:00:00Z/P1D"); - announceSegmentForServer(druidServer, segment, zkPathsConfig, jsonMapper); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - - TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); - List serverLookupRes = (List) timeline.lookup( - intervals - ); - Assert.assertEquals(1, serverLookupRes.size()); - - TimelineObjectHolder actualTimelineObjectHolder = serverLookupRes.get(0); - Assert.assertEquals(intervals, actualTimelineObjectHolder.getInterval()); - Assert.assertEquals("v1", actualTimelineObjectHolder.getVersion()); - - PartitionHolder actualPartitionHolder = actualTimelineObjectHolder.getObject(); - Assert.assertTrue(actualPartitionHolder.isComplete()); - Assert.assertEquals(1, Iterables.size(actualPartitionHolder)); - - SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject(); - Assert.assertFalse(segmentLoadInfo.isEmpty()); - Assert.assertEquals( - druidServer.getMetadata(), - Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers()) - ); - Assert.assertNotNull(timeline.findChunk(intervals, "v1", partition)); - - unannounceSegmentForServer(druidServer, segment); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - - Assert.assertEquals( - 0, - ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() - ); - Assert.assertNull(timeline.findChunk(intervals, "v1", partition)); - } - - @Test - public void testMultipleServerAddedRemovedSegment() throws Exception - { - segmentViewInitLatch = new CountDownLatch(1); - segmentAddedLatch = new CountDownLatch(5); - - // temporarily set latch count to 1 - segmentRemovedLatch = new CountDownLatch(1); - - setupViews(); - - final List druidServers = Lists.transform( - ImmutableList.of("localhost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), - new Function() - { - @Override - public DruidServer apply(String input) - { - return new DruidServer( - input, - input, - null, - 10000000L, - ServerType.HISTORICAL, - "default_tier", - 0 - ); - } - } - ); - - for (DruidServer druidServer : druidServers) { - setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper); - } - - final List segments = Lists.transform( - ImmutableList.of( - Pair.of("2011-04-01/2011-04-03", "v1"), - Pair.of("2011-04-03/2011-04-06", "v1"), - Pair.of("2011-04-01/2011-04-09", "v2"), - Pair.of("2011-04-06/2011-04-09", "v3"), - Pair.of("2011-04-01/2011-04-02", "v3") - ), new Function, DataSegment>() - { - @Override - public DataSegment apply(Pair input) - { - return dataSegmentWithIntervalAndVersion(input.lhs, input.rhs); - } - } - ); - - for (int i = 0; i < 5; ++i) { - announceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig, jsonMapper); - } - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - - TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); - assertValues( - Arrays.asList( - createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), - createExpected("2011-04-02/2011-04-06", "v2", druidServers.get(2), segments.get(2)), - createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3)) - ), - (List) timeline.lookup( - Intervals.of( - "2011-04-01/2011-04-09" - ) - ) - ); - - // unannounce the segment created by dataSegmentWithIntervalAndVersion("2011-04-01/2011-04-09", "v2") - unannounceSegmentForServer(druidServers.get(2), segments.get(2)); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - - // renew segmentRemovedLatch since we still have 4 segments to unannounce - segmentRemovedLatch = new CountDownLatch(4); - - timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); - assertValues( - Arrays.asList( - createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), - createExpected("2011-04-02/2011-04-03", "v1", druidServers.get(0), segments.get(0)), - createExpected("2011-04-03/2011-04-06", "v1", druidServers.get(1), segments.get(1)), - createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3)) - ), - (List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")) - ); - - // unannounce all the segments - for (int i = 0; i < 5; ++i) { - // skip the one that was previously unannounced - if (i != 2) { - unannounceSegmentForServer(druidServers.get(i), segments.get(i)); - } - } - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - - Assert.assertEquals( - 0, - ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() - ); - } - - private void unannounceSegmentForServer(DruidServer druidServer, DataSegment segment) throws Exception - { - curator - .delete() - .guaranteed() - .forPath(ZKPaths.makePath(inventoryPath, druidServer.getHost(), segment.getId().toString())); - } - - private Pair>> createExpected( - String intervalStr, - String version, - DruidServer druidServer, - DataSegment segment - ) - { - return Pair.of(Intervals.of(intervalStr), Pair.of(version, Pair.of(druidServer, segment))); - } - - private void assertValues( - List>>> expected, List actual - ) - { - Assert.assertEquals(expected.size(), actual.size()); - - for (int i = 0; i < expected.size(); ++i) { - Pair>> expectedPair = expected.get(i); - TimelineObjectHolder actualTimelineObjectHolder = actual.get(i); - - Assert.assertEquals(expectedPair.lhs, actualTimelineObjectHolder.getInterval()); - Assert.assertEquals(expectedPair.rhs.lhs, actualTimelineObjectHolder.getVersion()); - - PartitionHolder actualPartitionHolder = actualTimelineObjectHolder.getObject(); - Assert.assertTrue(actualPartitionHolder.isComplete()); - Assert.assertEquals(1, Iterables.size(actualPartitionHolder)); - - SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject(); - Assert.assertFalse(segmentLoadInfo.isEmpty()); - Assert.assertEquals(expectedPair.rhs.rhs.lhs.getMetadata(), - Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers())); - } - } - - private void setupViews() throws Exception - { - baseView = new BatchServerInventoryView( - zkPathsConfig, - curator, - jsonMapper, - Predicates.alwaysTrue(), - "test" - ) - { - @Override - public void registerSegmentCallback(Executor exec, final SegmentCallback callback) - { - super.registerSegmentCallback( - exec, - new SegmentCallback() - { - @Override - public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) - { - CallbackAction res = callback.segmentAdded(server, segment); - segmentAddedLatch.countDown(); - return res; - } - - @Override - public CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) - { - CallbackAction res = callback.segmentRemoved(server, segment); - segmentRemovedLatch.countDown(); - return res; - } - - @Override - public CallbackAction segmentViewInitialized() - { - CallbackAction res = callback.segmentViewInitialized(); - segmentViewInitLatch.countDown(); - return res; - } - } - ); - } - }; - - overlordServerView = new CoordinatorServerView( - baseView, - new CoordinatorSegmentWatcherConfig(), - new NoopServiceEmitter() - ); - - baseView.start(); - overlordServerView.start(); - } - - private DataSegment dataSegmentWithIntervalAndVersion(String intervalStr, String version) - { - return DataSegment.builder() - .dataSource("test_overlord_server_view") - .interval(Intervals.of(intervalStr)) - .loadSpec( - ImmutableMap.of( - "type", - "local", - "path", - "somewhere" - ) - ) - .version(version) - .dimensions(ImmutableList.of()) - .metrics(ImmutableList.of()) - .shardSpec(NoneShardSpec.instance()) - .binaryVersion(9) - .size(0) - .build(); - } - - @After - public void tearDown() throws Exception - { - baseView.stop(); - tearDownServerAndCurator(); - } -} diff --git a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java similarity index 81% rename from server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java index fccbd1651df8..402113e051d4 100644 --- a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java @@ -20,8 +20,6 @@ package org.apache.druid.client; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.fasterxml.jackson.dataformat.smile.SmileGenerator; import com.google.common.base.Function; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; @@ -32,7 +30,6 @@ import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; import org.apache.druid.client.selector.RandomServerSelectorStrategy; import org.apache.druid.curator.CuratorTestBase; -import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.http.client.HttpClient; @@ -61,20 +58,18 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class QueryableCoordinatorServerViewTest extends CuratorTestBase +public class CoordinatorTimelineTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; private final String inventoryPath; - private CountDownLatch segmentViewInitLatch; private CountDownLatch segmentAddedLatch; private CountDownLatch segmentRemovedLatch; private BatchServerInventoryView baseView; - private QueryableCoordinatorServerView overlordServerView; - public QueryableCoordinatorServerViewTest() + public CoordinatorTimelineTest() { jsonMapper = TestHelper.makeJsonMapper(); zkPathsConfig = new ZkPathsConfig(); @@ -90,7 +85,7 @@ public void setUp() throws Exception } @Test - public void testSingleServerAddedRemovedSegment() throws Exception + public void testSingleServerAddedRemovedSegment_CoordinatorServerView() throws Exception { segmentViewInitLatch = new CountDownLatch(1); segmentAddedLatch = new CountDownLatch(1); @@ -98,6 +93,46 @@ public void testSingleServerAddedRemovedSegment() throws Exception setupViews(); + CoordinatorServerView coordinatorServerView = new CoordinatorServerView( + baseView, + new CoordinatorSegmentWatcherConfig(), + new NoopServiceEmitter() + ); + + baseView.start(); + coordinatorServerView.start(); + + testSingleServerAddedRemovedSegment(coordinatorServerView); + } + + @Test + public void testSingleServerAddedRemovedSegment_QueryableCoordinatorServerView() throws Exception + { + segmentViewInitLatch = new CountDownLatch(1); + segmentAddedLatch = new CountDownLatch(1); + segmentRemovedLatch = new CountDownLatch(1); + + setupViews(); + + QueryableCoordinatorServerView queryableCoordinatorServerView = new QueryableCoordinatorServerView( + EasyMock.createMock(QueryToolChestWarehouse.class), + EasyMock.createMock(QueryWatcher.class), + new ObjectMapper(), + EasyMock.createMock(HttpClient.class), + baseView, + new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), + new NoopServiceEmitter(), + new CoordinatorSegmentWatcherConfig() + ); + + baseView.start(); + queryableCoordinatorServerView.start(); + + testSingleServerAddedRemovedSegment(queryableCoordinatorServerView); + } + + private void testSingleServerAddedRemovedSegment(CoordinatorTimeline coordinatorTimeline) throws Exception + { final DruidServer druidServer = new DruidServer( "localhost:1234", "localhost:1234", @@ -117,7 +152,7 @@ public void testSingleServerAddedRemovedSegment() throws Exception Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + TimelineLookup timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); List serverLookupRes = (List) timeline.lookup( intervals ); @@ -142,7 +177,7 @@ public void testSingleServerAddedRemovedSegment() throws Exception unannounceSegmentForServer(druidServer, segment); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() @@ -151,16 +186,54 @@ public void testSingleServerAddedRemovedSegment() throws Exception } @Test - public void testMultipleServerAddedRemovedSegment() throws Exception + public void testMultipleServerAddedRemovedSegment_CoordinatorServerView() throws Exception { segmentViewInitLatch = new CountDownLatch(1); segmentAddedLatch = new CountDownLatch(5); + segmentRemovedLatch = new CountDownLatch(1); + + setupViews(); + + CoordinatorServerView coordinatorServerView = new CoordinatorServerView( + baseView, + new CoordinatorSegmentWatcherConfig(), + new NoopServiceEmitter() + ); + + baseView.start(); + coordinatorServerView.start(); - // temporarily set latch count to 1 + testMultipleServerAddedRemovedSegment(coordinatorServerView); + } + + @Test + public void testMultipleServerAddedRemovedSegment_QueryableCoordinatorServerView() throws Exception + { + segmentViewInitLatch = new CountDownLatch(1); + segmentAddedLatch = new CountDownLatch(5); segmentRemovedLatch = new CountDownLatch(1); setupViews(); + QueryableCoordinatorServerView queryableCoordinatorServerView = new QueryableCoordinatorServerView( + EasyMock.createMock(QueryToolChestWarehouse.class), + EasyMock.createMock(QueryWatcher.class), + new ObjectMapper(), + EasyMock.createMock(HttpClient.class), + baseView, + new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), + new NoopServiceEmitter(), + new CoordinatorSegmentWatcherConfig() + ); + + baseView.start(); + queryableCoordinatorServerView.start(); + + testMultipleServerAddedRemovedSegment(queryableCoordinatorServerView); + } + + void testMultipleServerAddedRemovedSegment(CoordinatorTimeline coordinatorTimeline) throws Exception + { final List druidServers = Lists.transform( ImmutableList.of("localhost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), new Function() @@ -208,7 +281,7 @@ public DataSegment apply(Pair input) Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + TimelineLookup timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); assertValues( Arrays.asList( createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), @@ -229,7 +302,8 @@ public DataSegment apply(Pair input) // renew segmentRemovedLatch since we still have 4 segments to unannounce segmentRemovedLatch = new CountDownLatch(4); - timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + + timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); assertValues( Arrays.asList( createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), @@ -249,8 +323,7 @@ public DataSegment apply(Pair input) } Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); - + timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() @@ -299,7 +372,7 @@ private void assertValues( } } - private void setupViews() throws Exception + private void setupViews() { baseView = new BatchServerInventoryView( zkPathsConfig, @@ -343,33 +416,8 @@ public CallbackAction segmentViewInitialized() ); } }; - - overlordServerView = new QueryableCoordinatorServerView( - EasyMock.createMock(QueryToolChestWarehouse.class), - EasyMock.createMock(QueryWatcher.class), - getSmileMapper(), - EasyMock.createMock(HttpClient.class), - baseView, - new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), - new NoopServiceEmitter(), - new CoordinatorSegmentWatcherConfig() - ); - - baseView.start(); - overlordServerView.start(); } - private ObjectMapper getSmileMapper() - { - final SmileFactory smileFactory = new SmileFactory(); - smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false); - smileFactory.delegateToTextual(true); - final ObjectMapper retVal = new DefaultObjectMapper(smileFactory, "broker"); - retVal.getFactory().setCodec(retVal); - return retVal; - } - - private DataSegment dataSegmentWithIntervalAndVersion(String intervalStr, String version) { return DataSegment.builder() From 10068b60a8f143aaebf67d86d17066dbef1e03ee Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 18:10:44 +0530 Subject: [PATCH 26/82] Fix SegmentStatusInClusterTest --- .../apache/druid/timeline/SegmentStatusInClusterTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java b/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java index 54190870e77c..82b77ddee2bc 100644 --- a/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java +++ b/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java @@ -49,6 +49,8 @@ public class SegmentStatusInClusterTest private static final ImmutableMap LOAD_SPEC = ImmutableMap.of("something", "or_other"); private static final boolean OVERSHADOWED = true; private static final Integer REPLICATION_FACTOR = 2; + private static final Long NUM_ROWS = 10L; + private static final boolean REALTIME = true; private static final int TEST_VERSION = 0x9; private static final SegmentStatusInCluster SEGMENT = createSegmentForTest(); @@ -76,7 +78,7 @@ private static SegmentStatusInCluster createSegmentForTest() 1 ); - return new SegmentStatusInCluster(dataSegment, OVERSHADOWED, REPLICATION_FACTOR, 10L, true); + return new SegmentStatusInCluster(dataSegment, OVERSHADOWED, REPLICATION_FACTOR, NUM_ROWS, REALTIME); } @Test @@ -87,7 +89,7 @@ public void testUnwrappedSegmentWithOvershadowedStatusDeserialization() throws E JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - Assert.assertEquals(12, objectMap.size()); + Assert.assertEquals(14, objectMap.size()); Assert.assertEquals("something", objectMap.get("dataSource")); Assert.assertEquals(INTERVAL.toString(), objectMap.get("interval")); Assert.assertEquals("1", objectMap.get("version")); @@ -99,6 +101,8 @@ public void testUnwrappedSegmentWithOvershadowedStatusDeserialization() throws E Assert.assertEquals(1, objectMap.get("size")); Assert.assertEquals(OVERSHADOWED, objectMap.get("overshadowed")); Assert.assertEquals(REPLICATION_FACTOR, objectMap.get("replicationFactor")); + Assert.assertEquals(NUM_ROWS.intValue(), objectMap.get("numRows")); + Assert.assertEquals(REALTIME, objectMap.get("realtime")); final String json = MAPPER.writeValueAsString(SEGMENT); From 032734af9394b634f0dd027b3a22cffaf68a1fea Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 18:15:18 +0530 Subject: [PATCH 27/82] Address intellij inspection --- .../druid/segment/metadata/SegmentMetadataCacheCommon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 19540e231fe7..a22eb4c71d99 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -423,7 +423,7 @@ public void checkNullDatasource(SegmentMetadataCache schema) throws IOException Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } - public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) throws InterruptedException + public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) { Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() From 80f442400e9a712f59a39a6330fc6f1f5309c195 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 18:27:33 +0530 Subject: [PATCH 28/82] Add undeclared dependency in server module --- server/pom.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/pom.xml b/server/pom.xml index 5ba0b170a9ed..16c0399c5dbc 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -49,6 +49,11 @@ ${project.parent.version} runtime + + com.google.http-client + google-http-client + runtime + jakarta.inject @@ -307,10 +312,10 @@ com.fasterxml.jackson.module jackson-module-guice - + org.apache.commons commons-lang3 - + From 33a8dd54d3375d62ce83a284b32a55262b538919 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 21:37:07 +0530 Subject: [PATCH 29/82] Remove enabled field from SegmentMetadataCacheConfig --- .../segment/metadata/SegmentMetadataCacheConfig.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 2ebcb0e53c0a..0feac95506d0 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -27,11 +27,8 @@ public class SegmentMetadataCacheConfig { @JsonProperty private boolean awaitInitializationOnStart = false; - @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); - @JsonProperty - private final boolean enabled = false; @JsonProperty private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = @@ -71,9 +68,4 @@ public Period getMetadataRefreshPeriod() { return metadataRefreshPeriod; } - - public boolean isEnabled() - { - return enabled; - } } From 7a7ca55298f1ad283610afb018adcbc8564f087d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 13 Sep 2023 14:58:12 +0530 Subject: [PATCH 30/82] Add class to manage druid table information in SegmentMetadataCache, add javadocs for classes --- .../QueryableCoordinatorServerView.java | 13 +-- .../client/coordinator/CoordinatorClient.java | 2 +- .../druid/client/selector/ServerSelector.java | 4 +- .../metadata/DataSourceInformation.java | 2 +- .../metadata/SegmentMetadataCache.java | 84 +++++++++++++------ .../metadata/SegmentMetadataCacheConfig.java | 6 ++ .../druid/server/http/MetadataResource.java | 8 +- .../metadata/SegmentMetadataCacheCommon.java | 12 --- .../metadata/SegmentMetadataCacheTest.java | 8 +- .../schema/BrokerSegmentMetadataCache.java | 68 ++++++++++----- .../BrokerSegmentMetadataCacheConfig.java | 14 ++++ .../calcite/schema/MetadataSegmentView.java | 2 +- .../PhysicalDatasourceMetadataBuilder.java | 20 +++-- .../BrokerSegmentMetadataCacheTest.java | 15 +++- 14 files changed, 178 insertions(+), 80 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 719562a73716..0f69f2699a7f 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -44,11 +44,12 @@ /** * ServerView of coordinator for the state of segments being loaded in the cluster. *
    - * This class simply extends {@link BrokerServerView} and implements methods from {@link CoordinatorTimeline} - * for backward compatibility. This newer implementation is primarily required by - * {@link org.apache.druid.segment.metadata.SegmentMetadataCache} which will run on the Coordinator. + * This class extends {@link BrokerServerView} and implements {@link CoordinatorTimeline}. + * The main distinction between this class and {@link CoordinatorServerView} is the maintenance of a timeline + * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline. *
    - * Once this class is stable {@link CoordinatorServerView} should be removed. + * A new timeline class (implementing {@link TimelineServerView}) is required for + * {@link org.apache.druid.segment.metadata.SegmentMetadataCache}, which will run on the Coordinator. */ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline @@ -78,8 +79,8 @@ public boolean isAwaitInitializationOnStart() } /** - * Internally this class maintains a timeline of {@link ServerSelector}. - * This method returns a newline of the object {@link SegmentLoadInfo}. + * This class maintains a timeline of {@link ServerSelector} objects. + * This method returns a new timeline of the object {@link SegmentLoadInfo}. * * @param dataSource dataSoruce * @return timeline for the given dataSource diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index 2f51ca2ec743..ac61176d50fa 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -48,7 +48,7 @@ public interface CoordinatorClient ListenableFuture> fetchUsedSegments(String dataSource, List intervals); /** - * Fetches information for the given dataSources. + * Retrieves detailed metadata information for the specified data sources, which includes {@code RowSignature}. */ ListenableFuture> fetchDataSourceInformation(Set datasources); diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 0050edfa8f81..616778987428 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -218,8 +218,8 @@ public boolean hasData() } /** - * This conversion is required to make the newer {@link org.apache.druid.client.QueryableCoordinatorServerView} - * implement methods from {@link org.apache.druid.client.CoordinatorTimeline} + * This conversion, allows {@link org.apache.druid.client.QueryableCoordinatorServerView} + * to implement methods from {@link org.apache.druid.client.CoordinatorTimeline}. * * @return {@link SegmentLoadInfo} */ diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index b8e8c7d5389c..1fe6534d923c 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -26,7 +26,7 @@ import java.util.Objects; /** - * Encapsulates information of a dataSource like schema. + * Encapsulates information about a dataSource, such as its schema. */ public class DataSourceInformation { diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index b1ceed6ca0f8..376d4582d5d4 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -92,10 +92,8 @@ import java.util.stream.StreamSupport; /** - * Broker-side cache of segment metadata which combines segments to identify - * datasources which become "tables" in Calcite. This cache provides the "physical" - * metadata about a datasource which is blended with catalog "logical" metadata - * to provide the final user-view of each datasource. + * Coordinator-side cache of segment metadata that combines segments to identify + * datasources.The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle public class SegmentMetadataCache @@ -121,10 +119,11 @@ public class SegmentMetadataCache private final ColumnTypeMergePolicy columnTypeMergePolicy; /** - * Map of DataSource -> DruidTable. - * This map can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. + * Manages tables of DataSourceInformation. This manager is used to retrieve and store + * information related to dataSources. + * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final TableManager tableManager = new TableManager<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -281,16 +280,6 @@ public ServerView.CallbackAction serverSegmentRemoved( ); } - protected void removeFromTable(String s) - { - tables.remove(s); - } - - protected boolean tablesContains(String s) - { - return tables.containsKey(s); - } - private void startCacheExec() { cacheExec.submit( @@ -426,15 +415,15 @@ public void refresh(final Set segmentsToRefresh, final Set da } } - public void rebuildDatasource(String dataSource) + protected void rebuildDatasource(String dataSource) { final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tables.remove(dataSource); + tableManager.removeFromTable(dataSource); return; } - final DataSourceInformation oldTable = tables.put(dataSource, druidTable); + final DataSourceInformation oldTable = tableManager.put(dataSource, druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -456,17 +445,17 @@ public void awaitInitialization() throws InterruptedException public DataSourceInformation getDatasource(String name) { - return tables.get(name); + return tableManager.get(name); } public Map getDataSourceInformationMap() { - return ImmutableMap.copyOf(tables); + return tableManager.getAll(); } public Set getDatasourceNames() { - return tables.keySet(); + return tableManager.getKeySet(); } @VisibleForTesting @@ -533,7 +522,7 @@ public void addSegment(final DruidServerMetadata server, final DataSegment segme } ); } - if (!tablesContains(segment.getDataSource())) { + if (!tableManager.tablesContains(segment.getDataSource())) { refreshImmediately = true; } @@ -564,7 +553,7 @@ public void removeSegment(final DataSegment segment) totalSegments--; } if (segmentsMap.isEmpty()) { - removeFromTable(segment.getDataSource()); + tableManager.removeFromTable(segment.getDataSource()); log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); return null; } else { @@ -1127,4 +1116,49 @@ public String toString() return NAME; } } + + /** + * A generic class for managing table. + * + * @param The type of data associated with the tables (e.g., DataSourceInformation, PhysicalDataSourceMetadata). + */ + public static class TableManager + { + private final ConcurrentMap tables = new ConcurrentHashMap<>(); + + public void removeFromTable(String s) + { + tables.remove(s); + } + + public boolean tablesContains(String s) + { + return tables.containsKey(s); + } + + public T get(String s) + { + return tables.get(s); + } + + public Set getKeySet() + { + return tables.keySet(); + } + + public Map getAll() + { + return ImmutableMap.copyOf(tables); + } + + public T put(String key, T value) + { + return tables.put(key, value); + } + + public void putAll(Map anotherTable) + { + tables.putAll(anotherTable); + } + } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 0feac95506d0..de11d2b491e3 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -23,10 +23,16 @@ import com.google.common.annotations.VisibleForTesting; import org.joda.time.Period; +/** + * Coordinator-side configuration class for customizing properties related to the SegmentMetadata cache. + */ public class SegmentMetadataCacheConfig { + // A flag indicating whether to wait for cache initialization during startup. @JsonProperty private boolean awaitInitializationOnStart = false; + + // Cache refresh interval. @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 447fac8784aa..c79962c0d111 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -28,7 +28,6 @@ import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.Segments; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DataSourceInformation; @@ -72,7 +71,6 @@ @Path("/druid/coordinator/v1/metadata") public class MetadataResource { - private final Logger log = new Logger(MetadataResource.class); private final SegmentsMetadataManager segmentsMetadataManager; private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; @@ -221,11 +219,13 @@ private Response getAllUsedSegmentsWithAdditionalDetails( replicationFactor, numRows, // published segment can't be realtime - false); + false + ); }); Stream finalSegments = segmentStatus; + // conditionally add realtime segments information if (null != includeRealtimeSegments && null != segmentMetadataCache) { final Stream realtimeSegmentStatus = segmentMetadataCache .getSegmentMetadataSnapshot() @@ -357,7 +357,7 @@ public Response getSegment( } /** - * Return schema for the given dataSources. + * API to fetch {@link DataSourceInformation} for the specified dataSources. */ @POST @Path("/dataSourceInformation") diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index a22eb4c71d99..5717ab476c6f 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -80,10 +80,8 @@ import java.io.File; import java.io.IOException; import java.util.EnumSet; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -347,16 +345,6 @@ QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) ); } - public void checkStaleDatasourceRefresh(SegmentMetadataCache schema) throws IOException - { - Set segments = new HashSet<>(); - Set datasources = new HashSet<>(); - datasources.add("wat"); - Assert.assertNull(schema.getDatasource("wat")); - schema.refresh(segments, datasources); - Assert.assertNull(schema.getDatasource("wat")); - } - public void checkRefreshShouldEmitMetrics( SegmentMetadataCache schema, String dataSource, diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index e69555ee96b9..cd664f13504e 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -49,6 +49,7 @@ import org.junit.Test; import java.io.IOException; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -905,7 +906,12 @@ public void testSegmentMetadataFallbackType() public void testStaleDatasourceRefresh() throws IOException, InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkStaleDatasourceRefresh(schema); + Set segments = new HashSet<>(); + Set datasources = new HashSet<>(); + datasources.add("wat"); + Assert.assertNull(schema.getDatasource("wat")); + schema.refresh(segments, datasources); + Assert.assertNull(schema.getDatasource("wat")); } @Test diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index f59358586cde..90fdec9c54eb 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -21,6 +21,7 @@ import com.google.common.collect.Sets; import com.google.inject.Inject; +import io.vavr.Predicates; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.CoordinatorClient; @@ -32,7 +33,7 @@ import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; -import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; import org.apache.druid.timeline.SegmentId; import java.io.IOException; @@ -40,20 +41,34 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; /** - * + * Broker-side cache of segment metadata that combines segments to identify + * datasources which become "tables" in Calcite. This cache provides the "physical" + * metadata about a datasource which is blended with catalog "logical" metadata + * to provide the final user-view of each datasource. + *

    + * This class extends {@link SegmentMetadataCache} and introduces following changes, + *

      + *
    • The refresh mechanism now includes polling the coordinator for dataSource schema, + * and falling back to running {@link org.apache.druid.query.metadata.metadata.SegmentMetadataQuery}.
      • + *
    • It builds and caches {@link PhysicalDatasourceMetadata} object as table + * schema instead of {@link DataSourceInformation}.
      • + *
    */ @ManageLifecycle public class BrokerSegmentMetadataCache extends SegmentMetadataCache { private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataCache.class); - private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + /** + * Manages tables of PhysicalDataSourceMetadata. This manager is used to retrieve and store + * information related to dataSources. + * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. + */ + private final TableManager tableManager = new TableManager<>(); private final CoordinatorClient coordinatorClient; @Inject @@ -80,6 +95,11 @@ public BrokerSegmentMetadataCache( this.coordinatorClient = coordinatorClient; } + /** + * Refreshes the set of segments in two steps: + * 1. Polls the coordinator for the dataSource schema to update the {@code tables}. + * 2. Refreshes the remaining set of segments by executing a SegmentMetadataQuery. + */ @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { @@ -87,7 +107,7 @@ public void refresh(final Set segmentsToRefresh, final Set da segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); - Map polledDataSourceMetadata = new HashMap<>(); + Map polledDataSourceMetadata = new HashMap<>(); // Fetch dataSource information from the Coordinator try { @@ -101,7 +121,10 @@ public void refresh(final Set segmentsToRefresh, final Set da log.warn(e, "Exception querying coordinator to fetch dataSourceInformation."); } - tables.putAll(polledDataSourceMetadata); + // remove any extra dataSources returned + polledDataSourceMetadata.keySet().removeIf(Predicates.not(dataSourcesToQuery::contains)); + + tableManager.putAll(polledDataSourceMetadata); // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); @@ -128,44 +151,47 @@ public void refresh(final Set segmentsToRefresh, final Set da } } + /** + * Build table schema and convert DataSourceInformation to PhysicalDatasourceMetadata. + */ @Override - public void rebuildDatasource(String dataSource) + protected void rebuildDatasource(String dataSource) { final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tables.remove(dataSource); + tableManager.removeFromTable(dataSource); return; } - final DatasourceTable.PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); - final DatasourceTable.PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); + final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); + final PhysicalDatasourceMetadata oldTable = tableManager.put(dataSource, physicalDatasourceMetadata); if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { - log.info("[%s] signature is unchanged.", dataSource); + log.debug("[%s] signature is unchanged.", dataSource); } } @Override public Set getDatasourceNames() { - return tables.keySet(); + return tableManager.getKeySet(); } - @Override - protected void removeFromTable(String s) + public PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) { - tables.remove(s); + return tableManager.get(name); } @Override - protected boolean tablesContains(String s) + public DataSourceInformation getDatasource(String name) { - return tables.containsKey(s); + throw new UnsupportedOperationException(); } - public DatasourceTable.PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) + @Override + public Map getDataSourceInformationMap() { - return tables.get(name); + throw new UnsupportedOperationException(); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index a51aea811e93..fc6d94da2eb7 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -23,14 +23,25 @@ import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.joda.time.Period; +/** + * Broker-side configuration class for managing segment polling from the Coordinator and + * customizing properties related to the SegmentMetadata cache. + *

    + * The property {@link #awaitInitializationOnStart} is overridden in this class with a default value + * of {@code true}, which differs from the parent class. This ensures that the SegmentMetadata cache is + * fully initialized before other startup processes proceed. + */ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig { + // A flag indicating whether to cache polled segments from the Coordinator. @JsonProperty private boolean metadataSegmentCacheEnable = true; + // Interval for polling segments from the coordinator. @JsonProperty private long metadataSegmentPollPeriod = 60000; + // A flag indicating whether to wait for cache initialization during startup. @JsonProperty private boolean awaitInitializationOnStart = true; @@ -58,6 +69,9 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } + /** + * This property is overriden on the broker, so that the cache initialization blocks startup. + */ @Override public boolean isAwaitInitializationOnStart() { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java index 4f8044c48d45..50a506d0e6f3 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java @@ -53,7 +53,7 @@ /** * This class polls the Coordinator in background to keep the latest segments. - * Provides {@link #getSegments()} for others to get segments in metadata store. + * Provides {@link #getSegments()} for others to get the segments. * * The difference between this class and {@link SegmentsMetadataManager} is that this class resides * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index 5b344621f1f8..abd6f6af4d43 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -22,12 +22,15 @@ import com.google.inject.Inject; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.TableDataSource; -import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; -import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; +/** + * Builds {@link PhysicalDatasourceMetadata} for a dataSource, including information about its schema, + * joinability, and broadcast status. + */ public class PhysicalDatasourceMetadataBuilder { private final JoinableFactory joinableFactory; @@ -40,10 +43,12 @@ public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, Segmen this.segmentManager = segmentManager; } - DatasourceTable.PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) + /** + * Builds physical metadata for the given data source. + */ + PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) { final String dataSource = dataSourceInformation.getDatasource(); - final RowSignature rowSignature = dataSourceInformation.getRowSignature(); final TableDataSource tableDataSource; // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing @@ -59,6 +64,11 @@ DatasourceTable.PhysicalDatasourceMetadata build(DataSourceInformation dataSourc } else { tableDataSource = new TableDataSource(dataSource); } - return new DatasourceTable.PhysicalDatasourceMetadata(tableDataSource, rowSignature, isJoinable, isBroadcast); + return new PhysicalDatasourceMetadata( + tableDataSource, + dataSourceInformation.getRowSignature(), + isJoinable, + isBroadcast + ); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 5480cd61f352..33c76b0d2265 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -83,6 +83,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -234,6 +235,9 @@ public void refresh( return runningSchema; } + /** + * Test the case when coordinator returns information for all the requested dataSources + */ @Test public void testGetAllDsSchemaFromCoordinator() throws InterruptedException { @@ -280,6 +284,10 @@ public ListenableFuture> fetchDataSourceInformation( Assert.assertEquals(foo3RowSignature, schema.getPhysicalDatasourceMetadata("foo3").rowSignature()); } + /** + * Test the case when Coordinator returns information for a subset of dataSources. + * Check if SegmentMetadataQuery is fired for segments of the remaining dataSources. + */ @Test public void testGetFewDsSchemaFromCoordinator() throws InterruptedException { @@ -700,7 +708,12 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception public void testStaleDatasourceRefresh() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkStaleDatasourceRefresh(schema); + Set segments = new HashSet<>(); + Set datasources = new HashSet<>(); + datasources.add("wat"); + Assert.assertNull(schema.getPhysicalDatasourceMetadata("wat")); + schema.refresh(segments, datasources); + Assert.assertNull(schema.getPhysicalDatasourceMetadata("wat")); } @Test From b9fb83dbd54345e191aa9b2c563e0567d70e731d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 13 Sep 2023 15:25:28 +0530 Subject: [PATCH 31/82] Minor refactoring in SegmentMetadataCache --- .../metadata/SegmentMetadataCache.java | 41 ++++++++----------- .../schema/BrokerSegmentMetadataCache.java | 35 ++++++---------- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 376d4582d5d4..fd59ff397b9d 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -93,7 +93,7 @@ /** * Coordinator-side cache of segment metadata that combines segments to identify - * datasources.The cache provides metadata about a dataSource, see {@link DataSourceInformation}. + * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle public class SegmentMetadataCache @@ -411,23 +411,18 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - rebuildDatasource(dataSource); - } - } - - protected void rebuildDatasource(String dataSource) - { - final DataSourceInformation druidTable = buildDruidTable(dataSource); - if (druidTable == null) { - log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tableManager.removeFromTable(dataSource); - return; - } - final DataSourceInformation oldTable = tableManager.put(dataSource, druidTable); - if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { - log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); - } else { - log.debug("[%s] signature is unchanged.", dataSource); + final DataSourceInformation druidTable = buildDruidTable(dataSource); + if (druidTable == null) { + log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + tableManager.removeFromTable(dataSource); + return; + } + final DataSourceInformation oldTable = tableManager.put(dataSource, druidTable); + if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + } else { + log.debug("[%s] signature is unchanged.", dataSource); + } } } @@ -716,7 +711,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin throw new ISE("'segments' must all match 'dataSource'!"); } - log.info("Refreshing metadata for dataSource[%s].", dataSource); + log.debug("Refreshing metadata for dataSource[%s].", dataSource); final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); @@ -742,7 +737,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin log.warn("Got analysis for segment [%s] we didn't ask for, ignoring.", analysis.getId()); } else { final RowSignature rowSignature = analysisToRowSignature(analysis); - log.info("Segment[%s] has signature[%s].", segmentId, rowSignature); + log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature); segmentMetadataInfo.compute( dataSource, (datasourceKey, dataSourceSegments) -> { @@ -794,7 +789,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin emitter.emit(builder.setMetric("metadatacache/refresh/time", refreshDurationMillis)); - log.info( + log.debug( "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", dataSource, refreshDurationMillis, @@ -934,7 +929,7 @@ public Sequence runSegmentMetadataQuery( } @VisibleForTesting - public static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) + static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) { final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); for (Map.Entry entry : analysis.getColumns().entrySet()) { @@ -971,7 +966,7 @@ public static RowSignature analysisToRowSignature(final SegmentAnalysis analysis * This method is not thread-safe and must be used only in unit tests. */ @VisibleForTesting - public void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) + void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) { final ConcurrentSkipListMap dataSourceSegments = segmentMetadataInfo .computeIfAbsent( diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 90fdec9c54eb..e23a396be7db 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -147,28 +147,19 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - rebuildDatasource(dataSource); - } - } - - /** - * Build table schema and convert DataSourceInformation to PhysicalDatasourceMetadata. - */ - @Override - protected void rebuildDatasource(String dataSource) - { - final DataSourceInformation druidTable = buildDruidTable(dataSource); - if (druidTable == null) { - log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tableManager.removeFromTable(dataSource); - return; - } - final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); - final PhysicalDatasourceMetadata oldTable = tableManager.put(dataSource, physicalDatasourceMetadata); - if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { - log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); - } else { - log.debug("[%s] signature is unchanged.", dataSource); + final DataSourceInformation druidTable = buildDruidTable(dataSource); + if (druidTable == null) { + log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + tableManager.removeFromTable(dataSource); + return; + } + final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); + final PhysicalDatasourceMetadata oldTable = tableManager.put(dataSource, physicalDatasourceMetadata); + if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + } else { + log.debug("[%s] signature is unchanged.", dataSource); + } } } From aa2bfe790b620dab635fad38de8398803284f7de Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 13 Sep 2023 19:36:54 +0530 Subject: [PATCH 32/82] Make SegmentMetadataCache generic --- ...ruidSchemaInternRowSignatureBenchmark.java | 3 +- .../metadata/DataSourceInformation.java | 19 ++--- .../metadata/SegmentMetadataCache.java | 71 ++++--------------- .../druid/server/http/MetadataResource.java | 4 +- .../metadata/SegmentMetadataCacheTest.java | 50 ++++++------- .../org/apache/druid/cli/CliCoordinator.java | 48 ++++++++++++- .../schema/BrokerSegmentMetadataCache.java | 43 +++-------- .../druid/sql/calcite/schema/DruidSchema.java | 2 +- .../PhysicalDatasourceMetadataBuilder.java | 2 +- .../sql/calcite/table/DatasourceTable.java | 37 +++++----- .../BrokerSegmentMetadataCacheTest.java | 32 ++++----- ...PhysicalDataSourceMetadataBuilderTest.java | 8 +-- 12 files changed, 146 insertions(+), 173 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 5f0634f283a4..c6a0360fea39 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -30,6 +30,7 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; @@ -68,7 +69,7 @@ public class DruidSchemaInternRowSignatureBenchmark { private SegmentMetadataCacheForBenchmark cache; - private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache + private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache { public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index 1fe6534d923c..f25c104203c5 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; import org.apache.druid.segment.column.RowSignature; import java.util.Objects; @@ -30,22 +31,22 @@ */ public class DataSourceInformation { - private final String datasource; + private final String dataSource; private final RowSignature rowSignature; @JsonCreator public DataSourceInformation( - @JsonProperty("datasource") String datasource, + @JsonProperty("datasource") String dataSource, @JsonProperty("rowSignature") RowSignature rowSignature) { - this.datasource = datasource; - this.rowSignature = rowSignature; + this.dataSource = Preconditions.checkNotNull(dataSource, "'dataSource' must be nonnull"); + this.rowSignature = Preconditions.checkNotNull(rowSignature, "rowSignature"); } @JsonProperty - public String getDatasource() + public String getDataSource() { - return datasource; + return dataSource; } @JsonProperty @@ -64,7 +65,7 @@ public boolean equals(Object o) return false; } DataSourceInformation that = (DataSourceInformation) o; - return Objects.equals(datasource, that.datasource) && Objects.equals( + return Objects.equals(dataSource, that.dataSource) && Objects.equals( rowSignature, that.rowSignature ); @@ -73,14 +74,14 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(datasource, rowSignature); + return Objects.hash(dataSource, rowSignature); } @Override public String toString() { return "DataSourceSchema{" + - "datasource='" + datasource + '\'' + + "dataSource='" + dataSource + '\'' + ", rowSignature=" + rowSignature + '}'; } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index fd59ff397b9d..2ff4640f1297 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -96,7 +96,7 @@ * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle -public class SegmentMetadataCache +public class SegmentMetadataCache { // Newest segments first, so they override older ones. private static final Comparator SEGMENT_ORDER = Comparator @@ -119,11 +119,10 @@ public class SegmentMetadataCache private final ColumnTypeMergePolicy columnTypeMergePolicy; /** - * Manages tables of DataSourceInformation. This manager is used to retrieve and store - * information related to dataSources. + * Map of dataSource -> DataSourceInformation * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final TableManager tableManager = new TableManager<>(); + protected final ConcurrentMap tables = new ConcurrentHashMap<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -378,6 +377,7 @@ private void startCacheExec() @LifecycleStart public void start() throws InterruptedException { + log.info("Starting SegmentMetadataCache."); startCacheExec(); if (config.isAwaitInitializationOnStart()) { @@ -414,10 +414,10 @@ public void refresh(final Set segmentsToRefresh, final Set da final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tableManager.removeFromTable(dataSource); + tables.remove(dataSource); return; } - final DataSourceInformation oldTable = tableManager.put(dataSource, druidTable); + final DataSourceInformation oldTable = tables.put(dataSource, (T) druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -438,19 +438,19 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } - public DataSourceInformation getDatasource(String name) + public T getDatasource(String name) { - return tableManager.get(name); + return tables.get(name); } - public Map getDataSourceInformationMap() + public Map getDataSourceInformationMap() { - return tableManager.getAll(); + return ImmutableMap.copyOf(tables); } public Set getDatasourceNames() { - return tableManager.getKeySet(); + return tables.keySet(); } @VisibleForTesting @@ -517,7 +517,7 @@ public void addSegment(final DruidServerMetadata server, final DataSegment segme } ); } - if (!tableManager.tablesContains(segment.getDataSource())) { + if (!tables.containsKey(segment.getDataSource())) { refreshImmediately = true; } @@ -548,7 +548,7 @@ public void removeSegment(final DataSegment segment) totalSegments--; } if (segmentsMap.isEmpty()) { - tableManager.removeFromTable(segment.getDataSource()); + tables.remove(segment.getDataSource()); log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); return null; } else { @@ -1111,49 +1111,4 @@ public String toString() return NAME; } } - - /** - * A generic class for managing table. - * - * @param The type of data associated with the tables (e.g., DataSourceInformation, PhysicalDataSourceMetadata). - */ - public static class TableManager - { - private final ConcurrentMap tables = new ConcurrentHashMap<>(); - - public void removeFromTable(String s) - { - tables.remove(s); - } - - public boolean tablesContains(String s) - { - return tables.containsKey(s); - } - - public T get(String s) - { - return tables.get(s); - } - - public Set getKeySet() - { - return tables.keySet(); - } - - public Map getAll() - { - return ImmutableMap.copyOf(tables); - } - - public T put(String key, T value) - { - return tables.put(key, value); - } - - public void putAll(Map anotherTable) - { - tables.putAll(anotherTable); - } - } } diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index c79962c0d111..7fe2cd76e129 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -75,7 +75,7 @@ public class MetadataResource private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; - private final @Nullable SegmentMetadataCache segmentMetadataCache; + private final @Nullable SegmentMetadataCache segmentMetadataCache; @Inject public MetadataResource( @@ -83,7 +83,7 @@ public MetadataResource( IndexerMetadataStorageCoordinator metadataStorageCoordinator, AuthorizerMapper authorizerMapper, DruidCoordinator coordinator, - @Nullable SegmentMetadataCache segmentMetadataCache + @Nullable SegmentMetadataCache segmentMetadataCache ) { this.segmentsMetadataManager = segmentsMetadataManager; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index cd664f13504e..78ba2add3627 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -63,7 +63,7 @@ public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - private SegmentMetadataCache runningSchema; + private SegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @@ -85,15 +85,15 @@ public void tearDown() throws Exception walker.close(); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException + public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException + public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException { Preconditions.checkState(runningSchema == null); - runningSchema = new SegmentMetadataCache( + runningSchema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, config, @@ -126,7 +126,7 @@ public void markDataSourceAsNeedRebuild(String datasource) @Test public void testGetTableMap() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); final Set tableNames = schema.getDatasourceNames(); @@ -136,7 +136,7 @@ public void testGetTableMap() throws InterruptedException @Test public void testGetTableMapFoo() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -164,7 +164,7 @@ public void testGetTableMapFoo() throws InterruptedException @Test public void testGetTableMapFoo2() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo2"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -185,7 +185,7 @@ public void testGetTableMapSomeTable() throws InterruptedException { // using 'newest first' column type merge strategy, the types are expected to be the types defined in the newer // segment, except for json, which is special handled - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( new SegmentMetadataCacheConfig() { @Override public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() @@ -232,7 +232,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt { // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); final RowSignature fooRowSignature = fooDs.getRowSignature(); @@ -276,28 +276,28 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt @Test public void testAvailableSegmentMetadataNumRows() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullAvailableSegmentMetadata(schema); } @Test public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() .stream() @@ -354,7 +354,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -395,7 +395,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -440,7 +440,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -482,7 +482,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -521,7 +521,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -577,7 +577,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -636,7 +636,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr { String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -669,7 +669,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -715,7 +715,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -785,7 +785,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. - SegmentMetadataCache mySchema = new SegmentMetadataCache( + SegmentMetadataCache mySchema = new SegmentMetadataCache( factoryMock, serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -905,7 +905,7 @@ public void testSegmentMetadataFallbackType() @Test public void testStaleDatasourceRefresh() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Set segments = new HashSet<>(); Set datasources = new HashSet<>(); datasources.add("wat"); @@ -920,7 +920,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 6dbd99d3c0d3..b111c1147cc8 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -74,6 +74,7 @@ import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory; import org.apache.druid.java.util.common.lifecycle.Lifecycle; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.metadata.MetadataRuleManager; import org.apache.druid.metadata.MetadataRuleManagerConfig; @@ -87,9 +88,11 @@ import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.ClientQuerySegmentWalker; +import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.CoordinatorConfigManager; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -125,6 +128,7 @@ import org.apache.druid.server.lookup.cache.LookupCoordinatorManagerConfig; import org.apache.druid.server.metrics.ServiceStatusMonitor; import org.apache.druid.server.router.TieredBrokerConfig; +import org.apache.druid.server.security.Escalator; import org.eclipse.jetty.server.Server; import org.joda.time.Duration; @@ -363,7 +367,6 @@ private boolean isSegmentMetadataCacheEnabled() return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED)); } - private static class CoordinatorCustomDutyGroupsProvider implements Provider { private Properties props; @@ -481,7 +484,48 @@ public void configure(Binder binder) binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, QueryableCoordinatorServerView.class); - LifecycleModule.register(binder, SegmentMetadataCache.class); + binder.bind(new TypeLiteral>() {}) + .toProvider(SegmentMetadataCacheProvider.class).in(ManageLifecycle.class); + } + + private static class SegmentMetadataCacheProvider implements Provider> + { + private final QueryLifecycleFactory queryLifecycleFactory; + private final TimelineServerView serverView; + private final SegmentMetadataCacheConfig config; + private final Escalator escalator; + private final InternalQueryConfig internalQueryConfig; + private final ServiceEmitter emitter; + + @Inject + public SegmentMetadataCacheProvider( + QueryLifecycleFactory queryLifecycleFactory, + TimelineServerView serverView, + SegmentMetadataCacheConfig config, + Escalator escalator, + InternalQueryConfig internalQueryConfig, + ServiceEmitter emitter + ) + { + this.queryLifecycleFactory = queryLifecycleFactory; + this.serverView = serverView; + this.config = config; + this.escalator = escalator; + this.internalQueryConfig = internalQueryConfig; + this.emitter = emitter; + } + + @Override + public SegmentMetadataCache get() + { + return new SegmentMetadataCache<>( + queryLifecycleFactory, + serverView, + config, + escalator, + internalQueryConfig, + emitter); + } } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index e23a396be7db..14803ea31417 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -44,9 +44,9 @@ /** * Broker-side cache of segment metadata that combines segments to identify - * datasources which become "tables" in Calcite. This cache provides the "physical" - * metadata about a datasource which is blended with catalog "logical" metadata - * to provide the final user-view of each datasource. + * dataSources which become "tables" in Calcite. This cache provides the "physical" + * metadata about a dataSource which is blended with catalog "logical" metadata + * to provide the final user-view of each dataSource. *

    * This class extends {@link SegmentMetadataCache} and introduces following changes, *

      @@ -57,7 +57,7 @@ *
    */ @ManageLifecycle -public class BrokerSegmentMetadataCache extends SegmentMetadataCache +public class BrokerSegmentMetadataCache extends SegmentMetadataCache { private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataCache.class); @@ -68,7 +68,7 @@ public class BrokerSegmentMetadataCache extends SegmentMetadataCache * information related to dataSources. * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final TableManager tableManager = new TableManager<>(); + //private final TableManager<> tableManager = new TableManager<>(); private final CoordinatorClient coordinatorClient; @Inject @@ -113,7 +113,7 @@ public void refresh(final Set segmentsToRefresh, final Set da try { FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) .forEach(item -> polledDataSourceMetadata.put( - item.getDatasource(), + item.getDataSource(), physicalDatasourceMetadataBuilder.build(item) )); } @@ -124,7 +124,7 @@ public void refresh(final Set segmentsToRefresh, final Set da // remove any extra dataSources returned polledDataSourceMetadata.keySet().removeIf(Predicates.not(dataSourcesToQuery::contains)); - tableManager.putAll(polledDataSourceMetadata); + tables.putAll(polledDataSourceMetadata); // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); @@ -150,39 +150,16 @@ public void refresh(final Set segmentsToRefresh, final Set da final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tableManager.removeFromTable(dataSource); + tables.remove(dataSource); return; } final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); - final PhysicalDatasourceMetadata oldTable = tableManager.put(dataSource, physicalDatasourceMetadata); - if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { + final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); + if (oldTable == null || !oldTable.getRowSignature().equals(physicalDatasourceMetadata.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { log.debug("[%s] signature is unchanged.", dataSource); } } } - - @Override - public Set getDatasourceNames() - { - return tableManager.getKeySet(); - } - - public PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) - { - return tableManager.get(name); - } - - @Override - public DataSourceInformation getDatasource(String name) - { - throw new UnsupportedOperationException(); - } - - @Override - public Map getDataSourceInformationMap() - { - throw new UnsupportedOperationException(); - } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index 69b96403aa82..580b9acbb520 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -55,7 +55,7 @@ public Table getTable(String name) if (druidSchemaManager != null) { return druidSchemaManager.getTable(name); } else { - DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentMetadataCache.getPhysicalDatasourceMetadata(name); + DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentMetadataCache.getDatasource(name); return dsMetadata == null ? null : new DatasourceTable(dsMetadata); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index abd6f6af4d43..c5922b066885 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -48,7 +48,7 @@ public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, Segmen */ PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) { - final String dataSource = dataSourceInformation.getDatasource(); + final String dataSource = dataSourceInformation.getDataSource(); final TableDataSource tableDataSource; // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/table/DatasourceTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/table/DatasourceTable.java index 8b6903133a95..eeac85a2c1ed 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/table/DatasourceTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/table/DatasourceTable.java @@ -26,6 +26,7 @@ import org.apache.druid.query.DataSource; import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.metadata.DataSourceInformation; import java.util.Objects; @@ -44,34 +45,28 @@ public class DatasourceTable extends DruidTable * published in the Coordinator. Used only for datasources, since only * datasources are computed from segments. */ - public static class PhysicalDatasourceMetadata + public static class PhysicalDatasourceMetadata extends DataSourceInformation { - private final TableDataSource dataSource; - private final RowSignature rowSignature; + private final TableDataSource tableDataSource; private final boolean joinable; private final boolean broadcast; public PhysicalDatasourceMetadata( - final TableDataSource dataSource, + final TableDataSource tableDataSource, final RowSignature rowSignature, final boolean isJoinable, final boolean isBroadcast ) { - this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); - this.rowSignature = Preconditions.checkNotNull(rowSignature, "rowSignature"); + super(tableDataSource.getName(), rowSignature); + this.tableDataSource = Preconditions.checkNotNull(tableDataSource, "dataSource"); this.joinable = isJoinable; this.broadcast = isBroadcast; } public TableDataSource dataSource() { - return dataSource; - } - - public RowSignature rowSignature() - { - return rowSignature; + return tableDataSource; } public boolean isJoinable() @@ -93,20 +88,20 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { + return false; + } PhysicalDatasourceMetadata that = (PhysicalDatasourceMetadata) o; - if (!Objects.equals(dataSource, that.dataSource)) { - return false; - } - return Objects.equals(rowSignature, that.rowSignature); + return Objects.equals(tableDataSource, that.tableDataSource); } @Override public int hashCode() { - int result = dataSource != null ? dataSource.hashCode() : 0; - result = 31 * result + (rowSignature != null ? rowSignature.hashCode() : 0); + int result = tableDataSource != null ? tableDataSource.hashCode() : 0; + result = 31 * result + super.hashCode(); return result; } @@ -114,8 +109,8 @@ public int hashCode() public String toString() { return "DatasourceMetadata{" + - "dataSource=" + dataSource + - ", rowSignature=" + rowSignature + + "dataSource=" + tableDataSource + + ", rowSignature=" + getRowSignature() + '}'; } } @@ -126,7 +121,7 @@ public DatasourceTable( final PhysicalDatasourceMetadata physicalMetadata ) { - super(physicalMetadata.rowSignature()); + super(physicalMetadata.getRowSignature()); this.physicalMetadata = physicalMetadata; } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 33c76b0d2265..f4b55fb5929a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -278,10 +278,10 @@ public ListenableFuture> fetchDataSourceInformation( final Set tableNames = schema.getDatasourceNames(); Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE, "foo3"), tableNames); - Assert.assertEquals(dataSource1RowSignature, schema.getPhysicalDatasourceMetadata(DATASOURCE1).rowSignature()); - Assert.assertEquals(dataSource2RowSignature, schema.getPhysicalDatasourceMetadata(DATASOURCE2).rowSignature()); - Assert.assertEquals(someDataSourceRowSignature, schema.getPhysicalDatasourceMetadata(SOME_DATASOURCE).rowSignature()); - Assert.assertEquals(foo3RowSignature, schema.getPhysicalDatasourceMetadata("foo3").rowSignature()); + Assert.assertEquals(dataSource1RowSignature, schema.getDatasource(DATASOURCE1).getRowSignature()); + Assert.assertEquals(dataSource2RowSignature, schema.getDatasource(DATASOURCE2).getRowSignature()); + Assert.assertEquals(someDataSourceRowSignature, schema.getDatasource(SOME_DATASOURCE).getRowSignature()); + Assert.assertEquals(foo3RowSignature, schema.getDatasource("foo3").getRowSignature()); } /** @@ -355,7 +355,7 @@ public void testGetTableMap() throws InterruptedException public void testGetTableMapFoo() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata("foo"); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource("foo"); final DruidTable fooTable = new DatasourceTable(fooDs); final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); final List fields = rowType.getFieldList(); @@ -385,7 +385,7 @@ public void testGetTableMapFoo() throws InterruptedException public void testGetTableMapFoo2() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata("foo2"); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource("foo2"); final DruidTable fooTable = new DatasourceTable(fooDs); final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); final List fields = rowType.getFieldList(); @@ -417,7 +417,7 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli }, new NoopCoordinatorClient() ); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource(CalciteTests.SOME_DATASOURCE); final DruidTable table = new DatasourceTable(fooDs); final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); final List fields = rowType.getFieldList(); @@ -460,7 +460,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource(CalciteTests.SOME_DATASOURCE); final DruidTable table = new DatasourceTable(fooDs); final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); final List fields = rowType.getFieldList(); @@ -541,7 +541,7 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte { BrokerSegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); @@ -573,7 +573,7 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); Assert.assertTrue(fooTable.dataSource() instanceof GlobalTableDataSource); @@ -594,7 +594,7 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); @@ -607,7 +607,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw { BrokerSegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); @@ -640,7 +640,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); // Should not be a GlobalTableDataSource for now, because isGlobal is couple with joinability. Ideally this will be @@ -662,7 +662,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); @@ -711,9 +711,9 @@ public void testStaleDatasourceRefresh() throws IOException, InterruptedExceptio Set segments = new HashSet<>(); Set datasources = new HashSet<>(); datasources.add("wat"); - Assert.assertNull(schema.getPhysicalDatasourceMetadata("wat")); + Assert.assertNull(schema.getDatasource("wat")); schema.refresh(segments, datasources); - Assert.assertNull(schema.getPhysicalDatasourceMetadata("wat")); + Assert.assertNull(schema.getDatasource("wat")); } @Test diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java index 987a6d1e0100..ccdbef25461b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -115,13 +115,13 @@ public void testBuild() DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build(foo); Assert.assertTrue(fooDs.isJoinable()); Assert.assertTrue(fooDs.isBroadcast()); - Assert.assertEquals(fooDs.dataSource().getName(), foo.getDatasource()); - Assert.assertEquals(fooDs.rowSignature(), foo.getRowSignature()); + Assert.assertEquals(fooDs.dataSource().getName(), foo.getDataSource()); + Assert.assertEquals(fooDs.getRowSignature(), foo.getRowSignature()); DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build(bar); Assert.assertFalse(barDs.isJoinable()); Assert.assertFalse(barDs.isBroadcast()); - Assert.assertEquals(barDs.dataSource().getName(), bar.getDatasource()); - Assert.assertEquals(barDs.rowSignature(), bar.getRowSignature()); + Assert.assertEquals(barDs.dataSource().getName(), bar.getDataSource()); + Assert.assertEquals(barDs.getRowSignature(), bar.getRowSignature()); } } From e97dcda911a9d32fc61853b6c5a46743369f9eee Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 00:04:21 +0530 Subject: [PATCH 33/82] Add a generic abstract class for segment metadata cache --- ...ruidSchemaInternRowSignatureBenchmark.java | 3 +- .../QueryableCoordinatorServerView.java | 13 +- .../AbstractSegmentMetadataCache.java | 1084 +++++++++++++++++ .../metadata/SegmentMetadataCache.java | 1054 +--------------- .../metadata/SegmentMetadataCacheConfig.java | 6 +- .../druid/server/http/MetadataResource.java | 4 +- .../SegmentDataCacheConcurrencyTest.java | 19 +- .../metadata/SegmentMetadataCacheCommon.java | 10 +- .../SegmentMetadataCacheConfigTest.java | 4 +- .../metadata/SegmentMetadataCacheTest.java | 64 +- .../org/apache/druid/cli/CliCoordinator.java | 50 +- .../schema/BrokerSegmentMetadataCache.java | 43 +- .../PhysicalDatasourceMetadataBuilder.java | 6 +- .../BrokerSegmentMetadataCacheConfigTest.java | 6 +- .../BrokerSegmentMetadataCacheTest.java | 14 +- ...PhysicalDataSourceMetadataBuilderTest.java | 43 +- 16 files changed, 1216 insertions(+), 1207 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index c6a0360fea39..5f0634f283a4 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -30,7 +30,6 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; @@ -69,7 +68,7 @@ public class DruidSchemaInternRowSignatureBenchmark { private SegmentMetadataCacheForBenchmark cache; - private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache + private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache { public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 0f69f2699a7f..1db03afa2f85 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -32,6 +32,7 @@ import org.apache.druid.query.DataSource; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.VersionedIntervalTimeline; @@ -43,13 +44,13 @@ /** * ServerView of coordinator for the state of segments being loaded in the cluster. - *
    - * This class extends {@link BrokerServerView} and implements {@link CoordinatorTimeline}. + * + *

    This class extends {@link BrokerServerView} and implements {@link CoordinatorTimeline}. * The main distinction between this class and {@link CoordinatorServerView} is the maintenance of a timeline - * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline. - *
    - * A new timeline class (implementing {@link TimelineServerView}) is required for - * {@link org.apache.druid.segment.metadata.SegmentMetadataCache}, which will run on the Coordinator. + * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline.

    + * + *

    A new timeline class (implementing {@link TimelineServerView}) is required for + * {@link AbstractSegmentMetadataCache}, which will run on the Coordinator.

    */ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java new file mode 100644 index 000000000000..aefc16b679b3 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -0,0 +1,1084 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.base.Stopwatch; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.errorprone.annotations.concurrent.GuardedBy; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.ServerView; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Yielder; +import org.apache.druid.java.util.common.guava.Yielders; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.lifecycle.LifecycleStop; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.query.DruidMetrics; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; +import org.apache.druid.query.metadata.metadata.ColumnAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.Types; +import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.Escalator; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * An abstract class that listens for segment change events and caches segment metadata. + * It periodically queries data nodes to fetch segment schemas and combines them to build a data source schema. + * + *

    This class is generic and is parameterized by a type {@code T} that extends {@link DataSourceInformation}.

    + * + *

    This class has an abstract method {@link #refresh(Set, Set)} which the child class must override + * with the logic to get segment schema.

    + * + * @param The type of information associated with the data source, which must extend {@link DataSourceInformation}. + */ +public abstract class AbstractSegmentMetadataCache +{ + // Newest segments first, so they override older ones. + private static final Comparator SEGMENT_ORDER = Comparator + .comparing((SegmentId segmentId) -> segmentId.getInterval().getStart()) + .reversed() + .thenComparing(Function.identity()); + + private static final EmittingLogger log = new EmittingLogger(AbstractSegmentMetadataCache.class); + private static final int MAX_SEGMENTS_PER_QUERY = 15000; + private static final long DEFAULT_NUM_ROWS = 0; + private static final Interner ROW_SIGNATURE_INTERNER = Interners.newWeakInterner(); + private final QueryLifecycleFactory queryLifecycleFactory; + private final SegmentMetadataCacheConfig config; + // Escalator, so we can attach an authentication result to queries we generate. + private final Escalator escalator; + + private final ExecutorService cacheExec; + private final ExecutorService callbackExec; + private final ServiceEmitter emitter; + private final ColumnTypeMergePolicy columnTypeMergePolicy; + + /** + * Map of dataSource and generic object extending DataSourceInformation. + * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. + */ + protected final ConcurrentMap tables = new ConcurrentHashMap<>(); + + /** + * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. + * Use SortedMap for segments so they are merged in deterministic order, from older to newer. + * + * This map is updated by these two threads. + * + * - {@link #callbackExec} can update it in {@link #addSegment}, {@link #removeServerSegment}, + * and {@link #removeSegment}. + * - {@link #cacheExec} can update it in {@link #refreshSegmentsForDataSource}. + * + * While it is being updated, this map is read by these two types of thread. + * + * - {@link #cacheExec} can iterate all {@link AvailableSegmentMetadata}s per datasource. + * See {@link #buildDruidTable}. + * - Query threads can create a snapshot of the entire map for processing queries on the system table. + * See {@link #getSegmentMetadataSnapshot()}. + * + * As the access pattern of this map is read-intensive, we should minimize the contention between writers and readers. + * Since there are two threads that can update this map at the same time, those writers should lock the inner map + * first and then lock the entry before it updates segment metadata. This can be done using + * {@link ConcurrentMap#compute} as below. Note that, if you need to update the variables guarded by {@link #lock} + * inside of compute(), you should get the lock before calling compute() to keep the function executed in compute() + * not expensive. + * + * 
    +   *   segmentMedataInfo.compute(
+   *     datasourceParam,
+   *     (datasource, segmentsMap) -> {
+   *       if (segmentsMap == null) return null;
+   *       else {
+   *         segmentsMap.compute(
+   *           segmentIdParam,
+   *           (segmentId, segmentMetadata) -> {
+   *             // update segmentMetadata
+   *           }
+   *         );
+   *         return segmentsMap;
+   *       }
+   *     }
+   *   );
+   *
    + * + * Readers can simply delegate the locking to the concurrent map and iterate map entries. + */ + private final ConcurrentHashMap> segmentMetadataInfo + = new ConcurrentHashMap<>(); + + // For awaitInitialization. + private final CountDownLatch initialized = new CountDownLatch(1); + + // All mutable segments. + @GuardedBy("lock") + private final TreeSet mutableSegments = new TreeSet<>(SEGMENT_ORDER); + + // Configured context to attach to internally generated queries. + private final InternalQueryConfig internalQueryConfig; + + @GuardedBy("lock") + private boolean refreshImmediately = false; + + @GuardedBy("lock") + private boolean isServerViewInitialized = false; + + /** + * Counts the total number of known segments. This variable is used only for the segments table in the system schema + * to initialize a map with a more proper size when it creates a snapshot. As a result, it doesn't have to be exact, + * and thus there is no concurrency control for this variable. + */ + private int totalSegments = 0; + + /** + * This lock coordinates the access from multiple threads to those variables guarded by this lock. + * Currently, there are 2 threads that can access these variables. + * + * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceInformation} if necessary + * based on the information collected via timeline callbacks. + */ + protected final Object lock = new Object(); + + // All dataSources that need tables regenerated. + @GuardedBy("lock") + protected final Set dataSourcesNeedingRebuild = new HashSet<>(); + + // All segments that need to be refreshed. + @GuardedBy("lock") + protected final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); + + public AbstractSegmentMetadataCache( + final QueryLifecycleFactory queryLifecycleFactory, + final TimelineServerView serverView, + final SegmentMetadataCacheConfig config, + final Escalator escalator, + final InternalQueryConfig internalQueryConfig, + final ServiceEmitter emitter + ) + { + this.queryLifecycleFactory = Preconditions.checkNotNull(queryLifecycleFactory, "queryLifecycleFactory"); + Preconditions.checkNotNull(serverView, "serverView"); + this.config = Preconditions.checkNotNull(config, "config"); + this.columnTypeMergePolicy = config.getMetadataColumnTypeMergePolicy(); + this.cacheExec = Execs.singleThreaded("DruidSchema-Cache-%d"); + this.callbackExec = Execs.singleThreaded("DruidSchema-Callback-%d"); + this.escalator = escalator; + this.internalQueryConfig = internalQueryConfig; + this.emitter = emitter; + + initServerViewTimelineCallback(serverView); + } + + private void initServerViewTimelineCallback(final TimelineServerView serverView) + { + serverView.registerTimelineCallback( + callbackExec, + new TimelineServerView.TimelineCallback() + { + @Override + public ServerView.CallbackAction timelineInitialized() + { + synchronized (lock) { + isServerViewInitialized = true; + lock.notifyAll(); + } + + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentAdded(final DruidServerMetadata server, final DataSegment segment) + { + addSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DataSegment segment) + { + removeSegment(segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction serverSegmentRemoved( + final DruidServerMetadata server, + final DataSegment segment + ) + { + removeServerSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + } + + private void startCacheExec() + { + cacheExec.submit( + () -> { + long lastRefresh = 0L; + long lastFailure = 0L; + + try { + while (!Thread.currentThread().isInterrupted()) { + final Set segmentsToRefresh = new TreeSet<>(); + final Set dataSourcesToRebuild = new TreeSet<>(); + + try { + synchronized (lock) { + final long nextRefreshNoFuzz = DateTimes + .utc(lastRefresh) + .plus(config.getMetadataRefreshPeriod()) + .getMillis(); + + // Fuzz a bit to spread load out when we have multiple brokers. + final long nextRefresh = nextRefreshNoFuzz + (long) ((nextRefreshNoFuzz - lastRefresh) * 0.10); + + while (true) { + // Do not refresh if it's too soon after a failure (to avoid rapid cycles of failure). + final boolean wasRecentFailure = DateTimes.utc(lastFailure) + .plus(config.getMetadataRefreshPeriod()) + .isAfterNow(); + + if (isServerViewInitialized && + !wasRecentFailure && + (!segmentsNeedingRefresh.isEmpty() || !dataSourcesNeedingRebuild.isEmpty()) && + (refreshImmediately || nextRefresh < System.currentTimeMillis())) { + // We need to do a refresh. Break out of the waiting loop. + break; + } + + // lastFailure != 0L means exceptions happened before and there're some refresh work was not completed. + // so that even if ServerView is initialized, we can't let broker complete initialization. + if (isServerViewInitialized && lastFailure == 0L) { + // Server view is initialized, but we don't need to do a refresh. Could happen if there are + // no segments in the system yet. Just mark us as initialized, then. + initialized.countDown(); + } + + // Wait some more, we'll wake up when it might be time to do another refresh. + lock.wait(Math.max(1, nextRefresh - System.currentTimeMillis())); + } + + segmentsToRefresh.addAll(segmentsNeedingRefresh); + segmentsNeedingRefresh.clear(); + + // Mutable segments need a refresh every period, since new columns could be added dynamically. + segmentsNeedingRefresh.addAll(mutableSegments); + + lastFailure = 0L; + lastRefresh = System.currentTimeMillis(); + refreshImmediately = false; + } + + refresh(segmentsToRefresh, dataSourcesToRebuild); + + initialized.countDown(); + } + catch (InterruptedException e) { + // Fall through. + throw e; + } + catch (Exception e) { + log.warn(e, "Metadata refresh failed, trying again soon."); + + synchronized (lock) { + // Add our segments and dataSources back to their refresh and rebuild lists. + segmentsNeedingRefresh.addAll(segmentsToRefresh); + dataSourcesNeedingRebuild.addAll(dataSourcesToRebuild); + lastFailure = System.currentTimeMillis(); + } + } + } + } + catch (InterruptedException e) { + // Just exit. + } + catch (Throwable e) { + // Throwables that fall out to here (not caught by an inner try/catch) are potentially gnarly, like + // OOMEs. Anyway, let's just emit an alert and stop refreshing metadata. + log.makeAlert(e, "Metadata refresh failed permanently").emit(); + throw e; + } + finally { + log.info("Metadata refresh stopped."); + } + } + ); + } + + @LifecycleStart + public void start() throws InterruptedException + { + log.info("Starting SegmentMetadataCache."); + startCacheExec(); + + if (config.isAwaitInitializationOnStart()) { + final long startMillis = System.currentTimeMillis(); + log.info("%s waiting for initialization.", getClass().getSimpleName()); + awaitInitialization(); + final long endMillis = System.currentTimeMillis(); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); + emitter.emit(ServiceMetricEvent.builder().setMetric( + "metadatacache/init/time", + endMillis - startMillis + )); + } + } + + public abstract void refresh(Set segmentsToRefresh, Set dataSourcesToRebuild) throws IOException; + + @LifecycleStop + public void stop() + { + cacheExec.shutdownNow(); + callbackExec.shutdownNow(); + } + + public void awaitInitialization() throws InterruptedException + { + initialized.await(); + } + + public T getDatasource(String name) + { + return tables.get(name); + } + + public Map getDataSourceInformationMap() + { + return ImmutableMap.copyOf(tables); + } + + public Set getDatasourceNames() + { + return tables.keySet(); + } + + @VisibleForTesting + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + // Get lock first so that we won't wait in ConcurrentMap.compute(). + synchronized (lock) { + // someday we could hypothetically remove broker special casing, whenever BrokerServerView supports tracking + // broker served segments in the timeline, to ensure that removeSegment the event is triggered accurately + if (server.getType().equals(ServerType.BROKER)) { + // a segment on a broker means a broadcast datasource, skip metadata because we'll also see this segment on the + // historical, however mark the datasource for refresh because it needs to be globalized + markDataSourceAsNeedRebuild(segment.getDataSource()); + } else { + segmentMetadataInfo.compute( + segment.getDataSource(), + (datasource, segmentsMap) -> { + if (segmentsMap == null) { + segmentsMap = new ConcurrentSkipListMap<>(SEGMENT_ORDER); + } + segmentsMap.compute( + segment.getId(), + (segmentId, segmentMetadata) -> { + if (segmentMetadata == null) { + // Unknown segment. + totalSegments++; + // segmentReplicatable is used to determine if segments are served by historical or realtime servers + long isRealtime = server.isSegmentReplicationTarget() ? 0 : 1; + segmentMetadata = AvailableSegmentMetadata + .builder(segment, isRealtime, ImmutableSet.of(server), null, DEFAULT_NUM_ROWS) // Added without needing a refresh + .build(); + markSegmentAsNeedRefresh(segment.getId()); + if (!server.isSegmentReplicationTarget()) { + log.debug("Added new mutable segment [%s].", segment.getId()); + markSegmentAsMutable(segment.getId()); + } else { + log.debug("Added new immutable segment [%s].", segment.getId()); + } + } else { + // We know this segment. + final Set segmentServers = segmentMetadata.getReplicas(); + final ImmutableSet servers = new ImmutableSet.Builder() + .addAll(segmentServers) + .add(server) + .build(); + segmentMetadata = AvailableSegmentMetadata + .from(segmentMetadata) + .withReplicas(servers) + .withRealtime(recomputeIsRealtime(servers)) + .build(); + if (server.isSegmentReplicationTarget()) { + // If a segment shows up on a replicatable (historical) server at any point, then it must be immutable, + // even if it's also available on non-replicatable (realtime) servers. + unmarkSegmentAsMutable(segment.getId()); + log.debug("Segment[%s] has become immutable.", segment.getId()); + } + } + assert segmentMetadata != null; + return segmentMetadata; + } + ); + + return segmentsMap; + } + ); + } + if (!tables.containsKey(segment.getDataSource())) { + refreshImmediately = true; + } + + lock.notifyAll(); + } + } + + @VisibleForTesting + public void removeSegment(final DataSegment segment) + { + // Get lock first so that we won't wait in ConcurrentMap.compute(). + synchronized (lock) { + log.debug("Segment [%s] is gone.", segment.getId()); + + segmentsNeedingRefresh.remove(segment.getId()); + unmarkSegmentAsMutable(segment.getId()); + + segmentMetadataInfo.compute( + segment.getDataSource(), + (dataSource, segmentsMap) -> { + if (segmentsMap == null) { + log.warn("Unknown segment [%s] was removed from the cluster. Ignoring this event.", segment.getId()); + return null; + } else { + if (segmentsMap.remove(segment.getId()) == null) { + log.warn("Unknown segment [%s] was removed from the cluster. Ignoring this event.", segment.getId()); + } else { + totalSegments--; + } + if (segmentsMap.isEmpty()) { + tables.remove(segment.getDataSource()); + log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); + return null; + } else { + markDataSourceAsNeedRebuild(segment.getDataSource()); + return segmentsMap; + } + } + } + ); + + lock.notifyAll(); + } + } + + @VisibleForTesting + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + { + // Get lock first so that we won't wait in ConcurrentMap.compute(). + synchronized (lock) { + log.debug("Segment [%s] is gone from server [%s]", segment.getId(), server.getName()); + segmentMetadataInfo.compute( + segment.getDataSource(), + (datasource, knownSegments) -> { + if (knownSegments == null) { + log.warn( + "Unknown segment [%s] is removed from server [%s]. Ignoring this event", + segment.getId(), + server.getHost() + ); + return null; + } + + if (server.getType().equals(ServerType.BROKER)) { + // for brokers, if the segment drops from all historicals before the broker this could be null. + if (!knownSegments.isEmpty()) { + // a segment on a broker means a broadcast datasource, skip metadata because we'll also see this segment on the + // historical, however mark the datasource for refresh because it might no longer be broadcast or something + markDataSourceAsNeedRebuild(segment.getDataSource()); + } + } else { + knownSegments.compute( + segment.getId(), + (segmentId, segmentMetadata) -> { + if (segmentMetadata == null) { + log.warn( + "Unknown segment [%s] is removed from server [%s]. Ignoring this event", + segment.getId(), + server.getHost() + ); + return null; + } else { + final Set segmentServers = segmentMetadata.getReplicas(); + final ImmutableSet servers = FluentIterable + .from(segmentServers) + .filter(Predicates.not(Predicates.equalTo(server))) + .toSet(); + return AvailableSegmentMetadata + .from(segmentMetadata) + .withReplicas(servers) + .withRealtime(recomputeIsRealtime(servers)) + .build(); + } + } + ); + } + if (knownSegments.isEmpty()) { + return null; + } else { + return knownSegments; + } + } + ); + + lock.notifyAll(); + } + } + + private void markSegmentAsNeedRefresh(SegmentId segmentId) + { + synchronized (lock) { + segmentsNeedingRefresh.add(segmentId); + } + } + + private void markSegmentAsMutable(SegmentId segmentId) + { + synchronized (lock) { + mutableSegments.add(segmentId); + } + } + + private void unmarkSegmentAsMutable(SegmentId segmentId) + { + synchronized (lock) { + mutableSegments.remove(segmentId); + } + } + + @VisibleForTesting + public void markDataSourceAsNeedRebuild(String datasource) + { + synchronized (lock) { + dataSourcesNeedingRebuild.add(datasource); + } + } + + /** + * Attempt to refresh "segmentSignatures" for a set of segments. Returns the set of segments actually refreshed, + * which may be a subset of the asked-for set. + */ + @VisibleForTesting + public Set refreshSegments(final Set segments) throws IOException + { + final Set retVal = new HashSet<>(); + + // Organize segments by dataSource. + final Map> segmentMap = new TreeMap<>(); + + for (SegmentId segmentId : segments) { + segmentMap.computeIfAbsent(segmentId.getDataSource(), x -> new TreeSet<>(SEGMENT_ORDER)) + .add(segmentId); + } + + for (Map.Entry> entry : segmentMap.entrySet()) { + final String dataSource = entry.getKey(); + retVal.addAll(refreshSegmentsForDataSource(dataSource, entry.getValue())); + } + + return retVal; + } + + private long recomputeIsRealtime(ImmutableSet servers) + { + if (servers.isEmpty()) { + return 0; + } + final Optional historicalServer = servers + .stream() + // Ideally, this filter should have checked whether it's a broadcast segment loaded in brokers. + // However, we don't current track of the broadcast segments loaded in brokers, so this filter is still valid. + // See addSegment(), removeServerSegment(), and removeSegment() + .filter(metadata -> metadata.getType().equals(ServerType.HISTORICAL)) + .findAny(); + + // if there is any historical server in the replicas, isRealtime flag should be unset + return historicalServer.isPresent() ? 0 : 1; + } + + /** + * Attempt to refresh "segmentSignatures" for a set of segments for a particular dataSource. Returns the set of + * segments actually refreshed, which may be a subset of the asked-for set. + */ + private Set refreshSegmentsForDataSource(final String dataSource, final Set segments) + throws IOException + { + final Stopwatch stopwatch = Stopwatch.createStarted(); + + if (!segments.stream().allMatch(segmentId -> segmentId.getDataSource().equals(dataSource))) { + // Sanity check. We definitely expect this to pass. + throw new ISE("'segments' must all match 'dataSource'!"); + } + + log.debug("Refreshing metadata for dataSource[%s].", dataSource); + + final ServiceMetricEvent.Builder builder = + new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); + + emitter.emit(builder.setMetric("metadatacache/refresh/count", segments.size())); + + // Segment id string -> SegmentId object. + final Map segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString); + + final Set retVal = new HashSet<>(); + final Sequence sequence = runSegmentMetadataQuery( + Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY) + ); + + Yielder yielder = Yielders.each(sequence); + + try { + while (!yielder.isDone()) { + final SegmentAnalysis analysis = yielder.get(); + final SegmentId segmentId = segmentIdMap.get(analysis.getId()); + + if (segmentId == null) { + log.warn("Got analysis for segment [%s] we didn't ask for, ignoring.", analysis.getId()); + } else { + final RowSignature rowSignature = analysisToRowSignature(analysis); + log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature); + segmentMetadataInfo.compute( + dataSource, + (datasourceKey, dataSourceSegments) -> { + if (dataSourceSegments == null) { + // Datasource may have been removed or become unavailable while this refresh was ongoing. + log.warn( + "No segment map found with datasource [%s], skipping refresh of segment [%s]", + datasourceKey, + segmentId + ); + return null; + } else { + dataSourceSegments.compute( + segmentId, + (segmentIdKey, segmentMetadata) -> { + if (segmentMetadata == null) { + log.warn("No segment [%s] found, skipping refresh", segmentId); + return null; + } else { + final AvailableSegmentMetadata updatedSegmentMetadata = AvailableSegmentMetadata + .from(segmentMetadata) + .withRowSignature(rowSignature) + .withNumRows(analysis.getNumRows()) + .build(); + retVal.add(segmentId); + return updatedSegmentMetadata; + } + } + ); + + if (dataSourceSegments.isEmpty()) { + return null; + } else { + return dataSourceSegments; + } + } + } + ); + } + + yielder = yielder.next(null); + } + } + finally { + yielder.close(); + } + + long refreshDurationMillis = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + emitter.emit(builder.setMetric("metadatacache/refresh/time", refreshDurationMillis)); + + log.debug( + "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", + dataSource, + refreshDurationMillis, + retVal.size(), + segments.size() - retVal.size() + ); + + return retVal; + } + + @VisibleForTesting + @Nullable + public RowSignature buildDruidTable(final String dataSource) + { + ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); + + // Preserve order. + final Map columnTypes = new LinkedHashMap<>(); + + if (segmentsMap != null && !segmentsMap.isEmpty()) { + for (AvailableSegmentMetadata availableSegmentMetadata : segmentsMap.values()) { + final RowSignature rowSignature = availableSegmentMetadata.getRowSignature(); + if (rowSignature != null) { + for (String column : rowSignature.getColumnNames()) { + final ColumnType columnType = + rowSignature.getColumnType(column) + .orElseThrow(() -> new ISE("Encountered null type for column [%s]", column)); + + columnTypes.compute(column, (c, existingType) -> columnTypeMergePolicy.merge(existingType, columnType)); + } + } + } + } else { + // table has no segments + return null; + } + + final RowSignature.Builder builder = RowSignature.builder(); + columnTypes.forEach(builder::add); + + return builder.build(); + } + + public Map getSegmentMetadataSnapshot() + { + final Map segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments); + for (ConcurrentSkipListMap val : segmentMetadataInfo.values()) { + segmentMetadata.putAll(val); + } + return segmentMetadata; + } + + @Nullable + public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId) + { + if (!segmentMetadataInfo.containsKey(datasource)) { + return null; + } + return segmentMetadataInfo.get(datasource).get(segmentId); + } + + /** + * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. + * As a result, the returned value might be inexact. + */ + public int getTotalSegments() + { + return totalSegments; + } + + @VisibleForTesting + public TreeSet getSegmentsNeedingRefresh() + { + synchronized (lock) { + return segmentsNeedingRefresh; + } + } + + @VisibleForTesting + public TreeSet getMutableSegments() + { + synchronized (lock) { + return mutableSegments; + } + } + + @VisibleForTesting + public Set getDataSourcesNeedingRebuild() + { + synchronized (lock) { + return dataSourcesNeedingRebuild; + } + } + + /** + * Execute a SegmentMetadata query and return a {@link Sequence} of {@link SegmentAnalysis}. + * + * @param segments Iterable of {@link SegmentId} objects that are subject of the SegmentMetadata query. + * @return {@link Sequence} of {@link SegmentAnalysis} objects + */ + @VisibleForTesting + public Sequence runSegmentMetadataQuery( + final Iterable segments + ) + { + // Sanity check: getOnlyElement of a set, to ensure all segments have the same dataSource. + final String dataSource = Iterables.getOnlyElement( + StreamSupport.stream(segments.spliterator(), false) + .map(SegmentId::getDataSource).collect(Collectors.toSet()) + ); + + final MultipleSpecificSegmentSpec querySegmentSpec = new MultipleSpecificSegmentSpec( + StreamSupport.stream(segments.spliterator(), false) + .map(SegmentId::toDescriptor).collect(Collectors.toList()) + ); + + final SegmentMetadataQuery segmentMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(dataSource), + querySegmentSpec, + new AllColumnIncluderator(), + false, + // disable the parallel merge because we don't care about the merge and don't want to consume its resources + QueryContexts.override( + internalQueryConfig.getContext(), + QueryContexts.BROKER_PARALLEL_MERGE_KEY, + false + ), + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null // we don't care about merging strategy because merge is false + ); + + return queryLifecycleFactory + .factorize() + .runSimple(segmentMetadataQuery, escalator.createEscalatedAuthenticationResult(), Access.OK).getResults(); + } + + @VisibleForTesting + static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) + { + final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); + for (Map.Entry entry : analysis.getColumns().entrySet()) { + if (entry.getValue().isError()) { + // Skip columns with analysis errors. + continue; + } + + ColumnType valueType = entry.getValue().getTypeSignature(); + + // this shouldn't happen, but if it does, first try to fall back to legacy type information field in case + // standard upgrade order was not followed for 0.22 to 0.23+, and if that also fails, then assume types are some + // flavor of COMPLEX. + if (valueType == null) { + // at some point in the future this can be simplified to the contents of the catch clause here, once the + // likelyhood of upgrading from some version lower than 0.23 is low + try { + valueType = ColumnType.fromString(entry.getValue().getType()); + if (valueType == null) { + valueType = ColumnType.ofComplex(entry.getValue().getType()); + } + } + catch (IllegalArgumentException ignored) { + valueType = ColumnType.UNKNOWN_COMPLEX; + } + } + + rowSignatureBuilder.add(entry.getKey(), valueType); + } + return ROW_SIGNATURE_INTERNER.intern(rowSignatureBuilder.build()); + } + + /** + * This method is not thread-safe and must be used only in unit tests. + */ + @VisibleForTesting + void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) + { + final ConcurrentSkipListMap dataSourceSegments = segmentMetadataInfo + .computeIfAbsent( + segmentId.getDataSource(), + k -> new ConcurrentSkipListMap<>(SEGMENT_ORDER) + ); + if (dataSourceSegments.put(segmentId, availableSegmentMetadata) == null) { + totalSegments++; + } + } + + /** + * This is a helper method for unit tests to emulate heavy work done with {@link #lock}. + * It must be used only in unit tests. + */ + @VisibleForTesting + void doInLock(Runnable runnable) + { + synchronized (lock) { + runnable.run(); + } + } + + /** + * ColumnTypeMergePolicy defines the rules of which type to use when faced with the possibility of different types + * for the same column from segment to segment. It is used to help compute a {@link RowSignature} for a table in + * Druid based on the segment metadata of all segments, merging the types of each column encountered to end up with + * a single type to represent it globally. + */ + @FunctionalInterface + public interface ColumnTypeMergePolicy + { + ColumnType merge(ColumnType existingType, ColumnType newType); + + @JsonCreator + static ColumnTypeMergePolicy fromString(String type) + { + if (LeastRestrictiveTypeMergePolicy.NAME.equalsIgnoreCase(type)) { + return LeastRestrictiveTypeMergePolicy.INSTANCE; + } + if (FirstTypeMergePolicy.NAME.equalsIgnoreCase(type)) { + return FirstTypeMergePolicy.INSTANCE; + } + throw new IAE("Unknown type [%s]", type); + } + } + + /** + * Classic logic, we use the first type we encounter. This policy is effectively 'newest first' because we iterated + * segments starting from the most recent time chunk, so this typically results in the most recently used type being + * chosen, at least for systems that are continuously updated with 'current' data. + * + * Since {@link ColumnTypeMergePolicy} are used to compute the SQL schema, at least in systems using SQL schemas which + * are partially or fully computed by this cache, this merge policy can result in query time errors if incompatible + * types are mixed if the chosen type is more restrictive than the types of some segments. If data is likely to vary + * in type across segments, consider using {@link LeastRestrictiveTypeMergePolicy} instead. + */ + public static class FirstTypeMergePolicy implements ColumnTypeMergePolicy + { + public static final String NAME = "latestInterval"; + private static final FirstTypeMergePolicy INSTANCE = new FirstTypeMergePolicy(); + + @Override + public ColumnType merge(ColumnType existingType, ColumnType newType) + { + if (existingType == null) { + return newType; + } + if (newType == null) { + return existingType; + } + // if any are json, are all json + if (ColumnType.NESTED_DATA.equals(newType) || ColumnType.NESTED_DATA.equals(existingType)) { + return ColumnType.NESTED_DATA; + } + // "existing type" is the 'newest' type, since we iterate the segments list by newest start time + return existingType; + } + + @Override + public int hashCode() + { + return Objects.hash(NAME); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + return o != null && getClass() == o.getClass(); + } + + @Override + public String toString() + { + return NAME; + } + } + + /** + * Resolves types using {@link ColumnType#leastRestrictiveType(ColumnType, ColumnType)} to find the ColumnType that + * can best represent all data contained across all segments. + */ + public static class LeastRestrictiveTypeMergePolicy implements ColumnTypeMergePolicy + { + public static final String NAME = "leastRestrictive"; + private static final LeastRestrictiveTypeMergePolicy INSTANCE = new LeastRestrictiveTypeMergePolicy(); + + @Override + public ColumnType merge(ColumnType existingType, ColumnType newType) + { + try { + return ColumnType.leastRestrictiveType(existingType, newType); + } + catch (Types.IncompatibleTypeException incompatibleTypeException) { + // fall back to first encountered type if they are not compatible for some reason + return FirstTypeMergePolicy.INSTANCE.merge(existingType, newType); + } + } + + @Override + public int hashCode() + { + return Objects.hash(NAME); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + return o != null && getClass() == o.getClass(); + } + + @Override + public String toString() + { + return NAME; + } + } +} diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 2ff4640f1297..83c67532d6e2 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -19,381 +19,48 @@ package org.apache.druid.segment.metadata; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicates; -import com.google.common.base.Stopwatch; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Interner; -import com.google.common.collect.Interners; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.inject.Inject; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.client.ServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.IAE; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.java.util.common.guava.Yielder; -import org.apache.druid.java.util.common.guava.Yielders; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.lifecycle.LifecycleStop; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; -import org.apache.druid.query.metadata.metadata.ColumnAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; -import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; -import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; -import org.apache.druid.segment.column.Types; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; -import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Escalator; -import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import javax.annotation.Nullable; import java.io.IOException; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; /** * Coordinator-side cache of segment metadata that combines segments to identify * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle -public class SegmentMetadataCache +public class SegmentMetadataCache extends AbstractSegmentMetadataCache { - // Newest segments first, so they override older ones. - private static final Comparator SEGMENT_ORDER = Comparator - .comparing((SegmentId segmentId) -> segmentId.getInterval().getStart()) - .reversed() - .thenComparing(Function.identity()); - - private static final EmittingLogger log = new EmittingLogger(SegmentMetadataCache.class); - private static final int MAX_SEGMENTS_PER_QUERY = 15000; - private static final long DEFAULT_NUM_ROWS = 0; - private static final Interner ROW_SIGNATURE_INTERNER = Interners.newWeakInterner(); - private final QueryLifecycleFactory queryLifecycleFactory; - private final SegmentMetadataCacheConfig config; - // Escalator, so we can attach an authentication result to queries we generate. - private final Escalator escalator; - - private final ExecutorService cacheExec; - private final ExecutorService callbackExec; - private final ServiceEmitter emitter; - private final ColumnTypeMergePolicy columnTypeMergePolicy; - - /** - * Map of dataSource -> DataSourceInformation - * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. - */ - protected final ConcurrentMap tables = new ConcurrentHashMap<>(); - - /** - * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. - * Use SortedMap for segments so they are merged in deterministic order, from older to newer. - * - * This map is updated by these two threads. - * - * - {@link #callbackExec} can update it in {@link #addSegment}, {@link #removeServerSegment}, - * and {@link #removeSegment}. - * - {@link #cacheExec} can update it in {@link #refreshSegmentsForDataSource}. - * - * While it is being updated, this map is read by these two types of thread. - * - * - {@link #cacheExec} can iterate all {@link AvailableSegmentMetadata}s per datasource. - * See {@link #buildDruidTable}. - * - Query threads can create a snapshot of the entire map for processing queries on the system table. - * See {@link #getSegmentMetadataSnapshot()}. - * - * As the access pattern of this map is read-intensive, we should minimize the contention between writers and readers. - * Since there are two threads that can update this map at the same time, those writers should lock the inner map - * first and then lock the entry before it updates segment metadata. This can be done using - * {@link ConcurrentMap#compute} as below. Note that, if you need to update the variables guarded by {@link #lock} - * inside of compute(), you should get the lock before calling compute() to keep the function executed in compute() - * not expensive. - * - * 
    -   *   segmentMedataInfo.compute(
-   *     datasourceParam,
-   *     (datasource, segmentsMap) -> {
-   *       if (segmentsMap == null) return null;
-   *       else {
-   *         segmentsMap.compute(
-   *           segmentIdParam,
-   *           (segmentId, segmentMetadata) -> {
-   *             // update segmentMetadata
-   *           }
-   *         );
-   *         return segmentsMap;
-   *       }
-   *     }
-   *   );
-   *
    - * - * Readers can simply delegate the locking to the concurrent map and iterate map entries. - */ - private final ConcurrentHashMap> segmentMetadataInfo - = new ConcurrentHashMap<>(); - - // For awaitInitialization. - private final CountDownLatch initialized = new CountDownLatch(1); - - // All mutable segments. - @GuardedBy("lock") - private final TreeSet mutableSegments = new TreeSet<>(SEGMENT_ORDER); - - // Configured context to attach to internally generated queries. - private final InternalQueryConfig internalQueryConfig; - - @GuardedBy("lock") - private boolean refreshImmediately = false; - - @GuardedBy("lock") - private boolean isServerViewInitialized = false; - - /** - * Counts the total number of known segments. This variable is used only for the segments table in the system schema - * to initialize a map with a more proper size when it creates a snapshot. As a result, it doesn't have to be exact, - * and thus there is no concurrency control for this variable. - */ - private int totalSegments = 0; - - /** - * This lock coordinates the access from multiple threads to those variables guarded by this lock. - * Currently, there are 2 threads that can access these variables. - * - * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceInformation} if necessary - * based on the information collected via timeline callbacks. - */ - protected final Object lock = new Object(); - - // All dataSources that need tables regenerated. - @GuardedBy("lock") - protected final Set dataSourcesNeedingRebuild = new HashSet<>(); - - // All segments that need to be refreshed. - @GuardedBy("lock") - protected final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); + private static final EmittingLogger log = new EmittingLogger(AbstractSegmentMetadataCache.class); @Inject public SegmentMetadataCache( - final QueryLifecycleFactory queryLifecycleFactory, - final TimelineServerView serverView, - final SegmentMetadataCacheConfig config, - final Escalator escalator, - final InternalQueryConfig internalQueryConfig, - final ServiceEmitter emitter + QueryLifecycleFactory queryLifecycleFactory, + TimelineServerView serverView, + SegmentMetadataCacheConfig config, + Escalator escalator, + InternalQueryConfig internalQueryConfig, + ServiceEmitter emitter ) { - this.queryLifecycleFactory = Preconditions.checkNotNull(queryLifecycleFactory, "queryLifecycleFactory"); - Preconditions.checkNotNull(serverView, "serverView"); - this.config = Preconditions.checkNotNull(config, "config"); - this.columnTypeMergePolicy = config.getMetadataColumnTypeMergePolicy(); - this.cacheExec = Execs.singleThreaded("DruidSchema-Cache-%d"); - this.callbackExec = Execs.singleThreaded("DruidSchema-Callback-%d"); - this.escalator = escalator; - this.internalQueryConfig = internalQueryConfig; - this.emitter = emitter; - - initServerViewTimelineCallback(serverView); - } - - private void initServerViewTimelineCallback(final TimelineServerView serverView) - { - serverView.registerTimelineCallback( - callbackExec, - new TimelineServerView.TimelineCallback() - { - @Override - public ServerView.CallbackAction timelineInitialized() - { - synchronized (lock) { - isServerViewInitialized = true; - lock.notifyAll(); - } - - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentAdded(final DruidServerMetadata server, final DataSegment segment) - { - addSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentRemoved(final DataSegment segment) - { - removeSegment(segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction serverSegmentRemoved( - final DruidServerMetadata server, - final DataSegment segment - ) - { - removeServerSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - } - - private void startCacheExec() - { - cacheExec.submit( - () -> { - long lastRefresh = 0L; - long lastFailure = 0L; - - try { - while (!Thread.currentThread().isInterrupted()) { - final Set segmentsToRefresh = new TreeSet<>(); - final Set dataSourcesToRebuild = new TreeSet<>(); - - try { - synchronized (lock) { - final long nextRefreshNoFuzz = DateTimes - .utc(lastRefresh) - .plus(config.getMetadataRefreshPeriod()) - .getMillis(); - - // Fuzz a bit to spread load out when we have multiple brokers. - final long nextRefresh = nextRefreshNoFuzz + (long) ((nextRefreshNoFuzz - lastRefresh) * 0.10); - - while (true) { - // Do not refresh if it's too soon after a failure (to avoid rapid cycles of failure). - final boolean wasRecentFailure = DateTimes.utc(lastFailure) - .plus(config.getMetadataRefreshPeriod()) - .isAfterNow(); - - if (isServerViewInitialized && - !wasRecentFailure && - (!segmentsNeedingRefresh.isEmpty() || !dataSourcesNeedingRebuild.isEmpty()) && - (refreshImmediately || nextRefresh < System.currentTimeMillis())) { - // We need to do a refresh. Break out of the waiting loop. - break; - } - - // lastFailure != 0L means exceptions happened before and there're some refresh work was not completed. - // so that even if ServerView is initialized, we can't let broker complete initialization. - if (isServerViewInitialized && lastFailure == 0L) { - // Server view is initialized, but we don't need to do a refresh. Could happen if there are - // no segments in the system yet. Just mark us as initialized, then. - initialized.countDown(); - } - - // Wait some more, we'll wake up when it might be time to do another refresh. - lock.wait(Math.max(1, nextRefresh - System.currentTimeMillis())); - } - - segmentsToRefresh.addAll(segmentsNeedingRefresh); - segmentsNeedingRefresh.clear(); - - // Mutable segments need a refresh every period, since new columns could be added dynamically. - segmentsNeedingRefresh.addAll(mutableSegments); - - lastFailure = 0L; - lastRefresh = System.currentTimeMillis(); - refreshImmediately = false; - } - - refresh(segmentsToRefresh, dataSourcesToRebuild); - - initialized.countDown(); - } - catch (InterruptedException e) { - // Fall through. - throw e; - } - catch (Exception e) { - log.warn(e, "Metadata refresh failed, trying again soon."); - - synchronized (lock) { - // Add our segments and dataSources back to their refresh and rebuild lists. - segmentsNeedingRefresh.addAll(segmentsToRefresh); - dataSourcesNeedingRebuild.addAll(dataSourcesToRebuild); - lastFailure = System.currentTimeMillis(); - } - } - } - } - catch (InterruptedException e) { - // Just exit. - } - catch (Throwable e) { - // Throwables that fall out to here (not caught by an inner try/catch) are potentially gnarly, like - // OOMEs. Anyway, let's just emit an alert and stop refreshing metadata. - log.makeAlert(e, "Metadata refresh failed permanently").emit(); - throw e; - } - finally { - log.info("Metadata refresh stopped."); - } - } - ); - } - - @LifecycleStart - public void start() throws InterruptedException - { - log.info("Starting SegmentMetadataCache."); - startCacheExec(); - - if (config.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); - awaitInitialization(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().setMetric( - "metadatacache/init/time", - endMillis - startMillis - )); - } + super(queryLifecycleFactory, serverView, config, escalator, internalQueryConfig, emitter); } - @VisibleForTesting + /** + * Fires SegmentMetadataQuery to fetch schema information for each segment in the refresh list. + * The schema information for individual segments is combined to construct a table schema, which is then cached. + */ + @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { // Refresh the segments. @@ -411,13 +78,14 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - final DataSourceInformation druidTable = buildDruidTable(dataSource); - if (druidTable == null) { + final RowSignature rowSignature = buildDruidTable(dataSource); + if (rowSignature == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final DataSourceInformation oldTable = tables.put(dataSource, (T) druidTable); + DataSourceInformation druidTable = new DataSourceInformation(dataSource, rowSignature); + final DataSourceInformation oldTable = tables.put(dataSource, druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -425,690 +93,4 @@ public void refresh(final Set segmentsToRefresh, final Set da } } } - - @LifecycleStop - public void stop() - { - cacheExec.shutdownNow(); - callbackExec.shutdownNow(); - } - - public void awaitInitialization() throws InterruptedException - { - initialized.await(); - } - - public T getDatasource(String name) - { - return tables.get(name); - } - - public Map getDataSourceInformationMap() - { - return ImmutableMap.copyOf(tables); - } - - public Set getDatasourceNames() - { - return tables.keySet(); - } - - @VisibleForTesting - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - // Get lock first so that we won't wait in ConcurrentMap.compute(). - synchronized (lock) { - // someday we could hypothetically remove broker special casing, whenever BrokerServerView supports tracking - // broker served segments in the timeline, to ensure that removeSegment the event is triggered accurately - if (server.getType().equals(ServerType.BROKER)) { - // a segment on a broker means a broadcast datasource, skip metadata because we'll also see this segment on the - // historical, however mark the datasource for refresh because it needs to be globalized - markDataSourceAsNeedRebuild(segment.getDataSource()); - } else { - segmentMetadataInfo.compute( - segment.getDataSource(), - (datasource, segmentsMap) -> { - if (segmentsMap == null) { - segmentsMap = new ConcurrentSkipListMap<>(SEGMENT_ORDER); - } - segmentsMap.compute( - segment.getId(), - (segmentId, segmentMetadata) -> { - if (segmentMetadata == null) { - // Unknown segment. - totalSegments++; - // segmentReplicatable is used to determine if segments are served by historical or realtime servers - long isRealtime = server.isSegmentReplicationTarget() ? 0 : 1; - segmentMetadata = AvailableSegmentMetadata - .builder(segment, isRealtime, ImmutableSet.of(server), null, DEFAULT_NUM_ROWS) // Added without needing a refresh - .build(); - markSegmentAsNeedRefresh(segment.getId()); - if (!server.isSegmentReplicationTarget()) { - log.debug("Added new mutable segment [%s].", segment.getId()); - markSegmentAsMutable(segment.getId()); - } else { - log.debug("Added new immutable segment [%s].", segment.getId()); - } - } else { - // We know this segment. - final Set segmentServers = segmentMetadata.getReplicas(); - final ImmutableSet servers = new ImmutableSet.Builder() - .addAll(segmentServers) - .add(server) - .build(); - segmentMetadata = AvailableSegmentMetadata - .from(segmentMetadata) - .withReplicas(servers) - .withRealtime(recomputeIsRealtime(servers)) - .build(); - if (server.isSegmentReplicationTarget()) { - // If a segment shows up on a replicatable (historical) server at any point, then it must be immutable, - // even if it's also available on non-replicatable (realtime) servers. - unmarkSegmentAsMutable(segment.getId()); - log.debug("Segment[%s] has become immutable.", segment.getId()); - } - } - assert segmentMetadata != null; - return segmentMetadata; - } - ); - - return segmentsMap; - } - ); - } - if (!tables.containsKey(segment.getDataSource())) { - refreshImmediately = true; - } - - lock.notifyAll(); - } - } - - @VisibleForTesting - public void removeSegment(final DataSegment segment) - { - // Get lock first so that we won't wait in ConcurrentMap.compute(). - synchronized (lock) { - log.debug("Segment [%s] is gone.", segment.getId()); - - segmentsNeedingRefresh.remove(segment.getId()); - unmarkSegmentAsMutable(segment.getId()); - - segmentMetadataInfo.compute( - segment.getDataSource(), - (dataSource, segmentsMap) -> { - if (segmentsMap == null) { - log.warn("Unknown segment [%s] was removed from the cluster. Ignoring this event.", segment.getId()); - return null; - } else { - if (segmentsMap.remove(segment.getId()) == null) { - log.warn("Unknown segment [%s] was removed from the cluster. Ignoring this event.", segment.getId()); - } else { - totalSegments--; - } - if (segmentsMap.isEmpty()) { - tables.remove(segment.getDataSource()); - log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); - return null; - } else { - markDataSourceAsNeedRebuild(segment.getDataSource()); - return segmentsMap; - } - } - } - ); - - lock.notifyAll(); - } - } - - @VisibleForTesting - public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) - { - // Get lock first so that we won't wait in ConcurrentMap.compute(). - synchronized (lock) { - log.debug("Segment [%s] is gone from server [%s]", segment.getId(), server.getName()); - segmentMetadataInfo.compute( - segment.getDataSource(), - (datasource, knownSegments) -> { - if (knownSegments == null) { - log.warn( - "Unknown segment [%s] is removed from server [%s]. Ignoring this event", - segment.getId(), - server.getHost() - ); - return null; - } - - if (server.getType().equals(ServerType.BROKER)) { - // for brokers, if the segment drops from all historicals before the broker this could be null. - if (!knownSegments.isEmpty()) { - // a segment on a broker means a broadcast datasource, skip metadata because we'll also see this segment on the - // historical, however mark the datasource for refresh because it might no longer be broadcast or something - markDataSourceAsNeedRebuild(segment.getDataSource()); - } - } else { - knownSegments.compute( - segment.getId(), - (segmentId, segmentMetadata) -> { - if (segmentMetadata == null) { - log.warn( - "Unknown segment [%s] is removed from server [%s]. Ignoring this event", - segment.getId(), - server.getHost() - ); - return null; - } else { - final Set segmentServers = segmentMetadata.getReplicas(); - final ImmutableSet servers = FluentIterable - .from(segmentServers) - .filter(Predicates.not(Predicates.equalTo(server))) - .toSet(); - return AvailableSegmentMetadata - .from(segmentMetadata) - .withReplicas(servers) - .withRealtime(recomputeIsRealtime(servers)) - .build(); - } - } - ); - } - if (knownSegments.isEmpty()) { - return null; - } else { - return knownSegments; - } - } - ); - - lock.notifyAll(); - } - } - - private void markSegmentAsNeedRefresh(SegmentId segmentId) - { - synchronized (lock) { - segmentsNeedingRefresh.add(segmentId); - } - } - - private void markSegmentAsMutable(SegmentId segmentId) - { - synchronized (lock) { - mutableSegments.add(segmentId); - } - } - - private void unmarkSegmentAsMutable(SegmentId segmentId) - { - synchronized (lock) { - mutableSegments.remove(segmentId); - } - } - - @VisibleForTesting - public void markDataSourceAsNeedRebuild(String datasource) - { - synchronized (lock) { - dataSourcesNeedingRebuild.add(datasource); - } - } - - /** - * Attempt to refresh "segmentSignatures" for a set of segments. Returns the set of segments actually refreshed, - * which may be a subset of the asked-for set. - */ - @VisibleForTesting - public Set refreshSegments(final Set segments) throws IOException - { - final Set retVal = new HashSet<>(); - - // Organize segments by dataSource. - final Map> segmentMap = new TreeMap<>(); - - for (SegmentId segmentId : segments) { - segmentMap.computeIfAbsent(segmentId.getDataSource(), x -> new TreeSet<>(SEGMENT_ORDER)) - .add(segmentId); - } - - for (Map.Entry> entry : segmentMap.entrySet()) { - final String dataSource = entry.getKey(); - retVal.addAll(refreshSegmentsForDataSource(dataSource, entry.getValue())); - } - - return retVal; - } - - private long recomputeIsRealtime(ImmutableSet servers) - { - if (servers.isEmpty()) { - return 0; - } - final Optional historicalServer = servers - .stream() - // Ideally, this filter should have checked whether it's a broadcast segment loaded in brokers. - // However, we don't current track of the broadcast segments loaded in brokers, so this filter is still valid. - // See addSegment(), removeServerSegment(), and removeSegment() - .filter(metadata -> metadata.getType().equals(ServerType.HISTORICAL)) - .findAny(); - - // if there is any historical server in the replicas, isRealtime flag should be unset - return historicalServer.isPresent() ? 0 : 1; - } - - /** - * Attempt to refresh "segmentSignatures" for a set of segments for a particular dataSource. Returns the set of - * segments actually refreshed, which may be a subset of the asked-for set. - */ - private Set refreshSegmentsForDataSource(final String dataSource, final Set segments) - throws IOException - { - final Stopwatch stopwatch = Stopwatch.createStarted(); - - if (!segments.stream().allMatch(segmentId -> segmentId.getDataSource().equals(dataSource))) { - // Sanity check. We definitely expect this to pass. - throw new ISE("'segments' must all match 'dataSource'!"); - } - - log.debug("Refreshing metadata for dataSource[%s].", dataSource); - - final ServiceMetricEvent.Builder builder = - new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); - - emitter.emit(builder.setMetric("metadatacache/refresh/count", segments.size())); - - // Segment id string -> SegmentId object. - final Map segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString); - - final Set retVal = new HashSet<>(); - final Sequence sequence = runSegmentMetadataQuery( - Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY) - ); - - Yielder yielder = Yielders.each(sequence); - - try { - while (!yielder.isDone()) { - final SegmentAnalysis analysis = yielder.get(); - final SegmentId segmentId = segmentIdMap.get(analysis.getId()); - - if (segmentId == null) { - log.warn("Got analysis for segment [%s] we didn't ask for, ignoring.", analysis.getId()); - } else { - final RowSignature rowSignature = analysisToRowSignature(analysis); - log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature); - segmentMetadataInfo.compute( - dataSource, - (datasourceKey, dataSourceSegments) -> { - if (dataSourceSegments == null) { - // Datasource may have been removed or become unavailable while this refresh was ongoing. - log.warn( - "No segment map found with datasource [%s], skipping refresh of segment [%s]", - datasourceKey, - segmentId - ); - return null; - } else { - dataSourceSegments.compute( - segmentId, - (segmentIdKey, segmentMetadata) -> { - if (segmentMetadata == null) { - log.warn("No segment [%s] found, skipping refresh", segmentId); - return null; - } else { - final AvailableSegmentMetadata updatedSegmentMetadata = AvailableSegmentMetadata - .from(segmentMetadata) - .withRowSignature(rowSignature) - .withNumRows(analysis.getNumRows()) - .build(); - retVal.add(segmentId); - return updatedSegmentMetadata; - } - } - ); - - if (dataSourceSegments.isEmpty()) { - return null; - } else { - return dataSourceSegments; - } - } - } - ); - } - - yielder = yielder.next(null); - } - } - finally { - yielder.close(); - } - - long refreshDurationMillis = stopwatch.elapsed(TimeUnit.MILLISECONDS); - - emitter.emit(builder.setMetric("metadatacache/refresh/time", refreshDurationMillis)); - - log.debug( - "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", - dataSource, - refreshDurationMillis, - retVal.size(), - segments.size() - retVal.size() - ); - - return retVal; - } - - @VisibleForTesting - @Nullable - public DataSourceInformation buildDruidTable(final String dataSource) - { - ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); - - // Preserve order. - final Map columnTypes = new LinkedHashMap<>(); - - if (segmentsMap != null && !segmentsMap.isEmpty()) { - for (AvailableSegmentMetadata availableSegmentMetadata : segmentsMap.values()) { - final RowSignature rowSignature = availableSegmentMetadata.getRowSignature(); - if (rowSignature != null) { - for (String column : rowSignature.getColumnNames()) { - final ColumnType columnType = - rowSignature.getColumnType(column) - .orElseThrow(() -> new ISE("Encountered null type for column [%s]", column)); - - columnTypes.compute(column, (c, existingType) -> columnTypeMergePolicy.merge(existingType, columnType)); - } - } - } - } else { - // table has no segments - return null; - } - - final RowSignature.Builder builder = RowSignature.builder(); - columnTypes.forEach(builder::add); - - return new DataSourceInformation(dataSource, builder.build()); - } - - public Map getSegmentMetadataSnapshot() - { - final Map segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments); - for (ConcurrentSkipListMap val : segmentMetadataInfo.values()) { - segmentMetadata.putAll(val); - } - return segmentMetadata; - } - - @Nullable - public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId) - { - if (!segmentMetadataInfo.containsKey(datasource)) { - return null; - } - return segmentMetadataInfo.get(datasource).get(segmentId); - } - - /** - * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. - * As a result, the returned value might be inexact. - */ - public int getTotalSegments() - { - return totalSegments; - } - - @VisibleForTesting - public TreeSet getSegmentsNeedingRefresh() - { - synchronized (lock) { - return segmentsNeedingRefresh; - } - } - - @VisibleForTesting - public TreeSet getMutableSegments() - { - synchronized (lock) { - return mutableSegments; - } - } - - @VisibleForTesting - public Set getDataSourcesNeedingRebuild() - { - synchronized (lock) { - return dataSourcesNeedingRebuild; - } - } - - /** - * Execute a SegmentMetadata query and return a {@link Sequence} of {@link SegmentAnalysis}. - * - * @param segments Iterable of {@link SegmentId} objects that are subject of the SegmentMetadata query. - * @return {@link Sequence} of {@link SegmentAnalysis} objects - */ - @VisibleForTesting - public Sequence runSegmentMetadataQuery( - final Iterable segments - ) - { - // Sanity check: getOnlyElement of a set, to ensure all segments have the same dataSource. - final String dataSource = Iterables.getOnlyElement( - StreamSupport.stream(segments.spliterator(), false) - .map(SegmentId::getDataSource).collect(Collectors.toSet()) - ); - - final MultipleSpecificSegmentSpec querySegmentSpec = new MultipleSpecificSegmentSpec( - StreamSupport.stream(segments.spliterator(), false) - .map(SegmentId::toDescriptor).collect(Collectors.toList()) - ); - - final SegmentMetadataQuery segmentMetadataQuery = new SegmentMetadataQuery( - new TableDataSource(dataSource), - querySegmentSpec, - new AllColumnIncluderator(), - false, - // disable the parallel merge because we don't care about the merge and don't want to consume its resources - QueryContexts.override( - internalQueryConfig.getContext(), - QueryContexts.BROKER_PARALLEL_MERGE_KEY, - false - ), - EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), - false, - null, - null // we don't care about merging strategy because merge is false - ); - - return queryLifecycleFactory - .factorize() - .runSimple(segmentMetadataQuery, escalator.createEscalatedAuthenticationResult(), Access.OK).getResults(); - } - - @VisibleForTesting - static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) - { - final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); - for (Map.Entry entry : analysis.getColumns().entrySet()) { - if (entry.getValue().isError()) { - // Skip columns with analysis errors. - continue; - } - - ColumnType valueType = entry.getValue().getTypeSignature(); - - // this shouldn't happen, but if it does, first try to fall back to legacy type information field in case - // standard upgrade order was not followed for 0.22 to 0.23+, and if that also fails, then assume types are some - // flavor of COMPLEX. - if (valueType == null) { - // at some point in the future this can be simplified to the contents of the catch clause here, once the - // likelyhood of upgrading from some version lower than 0.23 is low - try { - valueType = ColumnType.fromString(entry.getValue().getType()); - if (valueType == null) { - valueType = ColumnType.ofComplex(entry.getValue().getType()); - } - } - catch (IllegalArgumentException ignored) { - valueType = ColumnType.UNKNOWN_COMPLEX; - } - } - - rowSignatureBuilder.add(entry.getKey(), valueType); - } - return ROW_SIGNATURE_INTERNER.intern(rowSignatureBuilder.build()); - } - - /** - * This method is not thread-safe and must be used only in unit tests. - */ - @VisibleForTesting - void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) - { - final ConcurrentSkipListMap dataSourceSegments = segmentMetadataInfo - .computeIfAbsent( - segmentId.getDataSource(), - k -> new ConcurrentSkipListMap<>(SEGMENT_ORDER) - ); - if (dataSourceSegments.put(segmentId, availableSegmentMetadata) == null) { - totalSegments++; - } - } - - /** - * This is a helper method for unit tests to emulate heavy work done with {@link #lock}. - * It must be used only in unit tests. - */ - @VisibleForTesting - void doInLock(Runnable runnable) - { - synchronized (lock) { - runnable.run(); - } - } - - - /** - * ColumnTypeMergePolicy defines the rules of which type to use when faced with the possibility of different types - * for the same column from segment to segment. It is used to help compute a {@link RowSignature} for a table in - * Druid based on the segment metadata of all segments, merging the types of each column encountered to end up with - * a single type to represent it globally. - */ - @FunctionalInterface - public interface ColumnTypeMergePolicy - { - ColumnType merge(ColumnType existingType, ColumnType newType); - - @JsonCreator - static ColumnTypeMergePolicy fromString(String type) - { - if (LeastRestrictiveTypeMergePolicy.NAME.equalsIgnoreCase(type)) { - return LeastRestrictiveTypeMergePolicy.INSTANCE; - } - if (FirstTypeMergePolicy.NAME.equalsIgnoreCase(type)) { - return FirstTypeMergePolicy.INSTANCE; - } - throw new IAE("Unknown type [%s]", type); - } - } - - /** - * Classic logic, we use the first type we encounter. This policy is effectively 'newest first' because we iterated - * segments starting from the most recent time chunk, so this typically results in the most recently used type being - * chosen, at least for systems that are continuously updated with 'current' data. - * - * Since {@link ColumnTypeMergePolicy} are used to compute the SQL schema, at least in systems using SQL schemas which - * are partially or fully computed by this cache, this merge policy can result in query time errors if incompatible - * types are mixed if the chosen type is more restrictive than the types of some segments. If data is likely to vary - * in type across segments, consider using {@link LeastRestrictiveTypeMergePolicy} instead. - */ - public static class FirstTypeMergePolicy implements ColumnTypeMergePolicy - { - public static final String NAME = "latestInterval"; - private static final FirstTypeMergePolicy INSTANCE = new FirstTypeMergePolicy(); - - @Override - public ColumnType merge(ColumnType existingType, ColumnType newType) - { - if (existingType == null) { - return newType; - } - if (newType == null) { - return existingType; - } - // if any are json, are all json - if (ColumnType.NESTED_DATA.equals(newType) || ColumnType.NESTED_DATA.equals(existingType)) { - return ColumnType.NESTED_DATA; - } - // "existing type" is the 'newest' type, since we iterate the segments list by newest start time - return existingType; - } - - @Override - public int hashCode() - { - return Objects.hash(NAME); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - return o != null && getClass() == o.getClass(); - } - - @Override - public String toString() - { - return NAME; - } - } - - /** - * Resolves types using {@link ColumnType#leastRestrictiveType(ColumnType, ColumnType)} to find the ColumnType that - * can best represent all data contained across all segments. - */ - public static class LeastRestrictiveTypeMergePolicy implements ColumnTypeMergePolicy - { - public static final String NAME = "leastRestrictive"; - private static final LeastRestrictiveTypeMergePolicy INSTANCE = new LeastRestrictiveTypeMergePolicy(); - - @Override - public ColumnType merge(ColumnType existingType, ColumnType newType) - { - try { - return ColumnType.leastRestrictiveType(existingType, newType); - } - catch (Types.IncompatibleTypeException incompatibleTypeException) { - // fall back to first encountered type if they are not compatible for some reason - return FirstTypeMergePolicy.INSTANCE.merge(existingType, newType); - } - } - - @Override - public int hashCode() - { - return Objects.hash(NAME); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - return o != null && getClass() == o.getClass(); - } - - @Override - public String toString() - { - return NAME; - } - } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index de11d2b491e3..5857619e3947 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -37,8 +37,8 @@ public class SegmentMetadataCacheConfig private Period metadataRefreshPeriod = new Period("PT1M"); @JsonProperty - private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = - new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); + private AbstractSegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = + new AbstractSegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); public static SegmentMetadataCacheConfig create() { @@ -65,7 +65,7 @@ public boolean isAwaitInitializationOnStart() return awaitInitializationOnStart; } - public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + public AbstractSegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() { return metadataColumnTypeMergePolicy; } diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 7fe2cd76e129..c79962c0d111 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -75,7 +75,7 @@ public class MetadataResource private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; - private final @Nullable SegmentMetadataCache segmentMetadataCache; + private final @Nullable SegmentMetadataCache segmentMetadataCache; @Inject public MetadataResource( @@ -83,7 +83,7 @@ public MetadataResource( IndexerMetadataStorageCoordinator metadataStorageCoordinator, AuthorizerMapper authorizerMapper, DruidCoordinator coordinator, - @Nullable SegmentMetadataCache segmentMetadataCache + @Nullable SegmentMetadataCache segmentMetadataCache ) { this.segmentsMetadataManager = segmentsMetadataManager; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 6a333a9dde02..1c9256da0fc8 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -47,6 +47,7 @@ import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; @@ -89,7 +90,7 @@ public class SegmentDataCacheConcurrencyTest extends SegmentMetadataCacheCommon private File tmpDir; private TestServerInventoryView inventoryView; private BrokerServerView serverView; - private SegmentMetadataCache schema; + private AbstractSegmentMetadataCache schema; private ExecutorService exec; @Before @@ -115,10 +116,10 @@ public void tearDown() throws Exception } /** - * This tests the contention between three components, {@link SegmentMetadataCache}, + * This tests the contention between three components, {@link AbstractSegmentMetadataCache}, * {@code InventoryView}, and {@link BrokerServerView}. It first triggers * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with - * {@link SegmentMetadataCache#lock}, {@link SegmentMetadataCache#buildDruidTable} + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} * is overridden to sleep before doing real work. While refreshing * {@code SegmentMetadataCache}, more new segments are added to * {@code InventoryView}, which triggers updates of {@code BrokerServerView}. @@ -140,7 +141,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV ) { @Override - public DataSourceInformation buildDruidTable(final String dataSource) + public RowSignature buildDruidTable(final String dataSource) { doInLock(() -> { try { @@ -224,11 +225,11 @@ public CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegme } /** - * This tests the contention between two methods of {@link SegmentMetadataCache}: - * {@link SegmentMetadataCache#refresh} and - * {@link SegmentMetadataCache#getSegmentMetadataSnapshot()}. It first triggers + * This tests the contention between two methods of {@link AbstractSegmentMetadataCache}: + * {@link AbstractSegmentMetadataCache#refresh} and + * {@link AbstractSegmentMetadataCache#getSegmentMetadataSnapshot()}. It first triggers * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with - * {@link SegmentMetadataCache#lock}, {@link SegmentMetadataCache#buildDruidTable} + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} * is overridden to sleep before doing real work. While refreshing * {@code SegmentMetadataCache}, {@code getSegmentMetadataSnapshot()} is continuously * called to mimic reading the segments table of SystemSchema. All these calls @@ -248,7 +249,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() ) { @Override - public DataSourceInformation buildDruidTable(final String dataSource) + public RowSignature buildDruidTable(final String dataSource) { doInLock(() -> { try { diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 5717ab476c6f..42f365b9af85 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -346,7 +346,7 @@ QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) } public void checkRefreshShouldEmitMetrics( - SegmentMetadataCache schema, + AbstractSegmentMetadataCache schema, String dataSource, StubServiceEmitter emitter, CountDownLatch addSegmentLatch @@ -366,7 +366,7 @@ public void checkRefreshShouldEmitMetrics( emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); } - public void checkNullAvailableSegmentMetadata(SegmentMetadataCache schema) throws IOException + public void checkNullAvailableSegmentMetadata(AbstractSegmentMetadataCache schema) throws IOException { final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); final List segments = segmentMetadatas.values() @@ -388,7 +388,7 @@ public void checkNullAvailableSegmentMetadata(SegmentMetadataCache schema) throw Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } - public void checkNullDatasource(SegmentMetadataCache schema) throws IOException + public void checkNullDatasource(AbstractSegmentMetadataCache schema) throws IOException { final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); final List segments = segmentMetadatas.values() @@ -411,7 +411,7 @@ public void checkNullDatasource(SegmentMetadataCache schema) throws IOException Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } - public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) + public void checkAvailableSegmentMetadataNumRows(AbstractSegmentMetadataCache schema) { Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() @@ -459,7 +459,7 @@ public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); } - public void checkRunSegmentMetadataQueryWithContext(SegmentMetadataCache schema, QueryLifecycleFactory factoryMock, QueryLifecycle lifecycleMock) + public void checkRunSegmentMetadataQueryWithContext(AbstractSegmentMetadataCache schema, QueryLifecycleFactory factoryMock, QueryLifecycle lifecycleMock) { Map queryContext = ImmutableMap.of( QueryContexts.PRIORITY_KEY, 5, diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java index d625229972fe..119191dc767f 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java @@ -51,7 +51,7 @@ public void testDefaultConfig() final SegmentMetadataCacheConfig config = provider.get(); Assert.assertFalse(config.isAwaitInitializationOnStart()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); - Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); + Assert.assertEquals(new AbstractSegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } @Test @@ -74,7 +74,7 @@ public void testCustomizedConfig() Assert.assertFalse(config.isAwaitInitializationOnStart()); Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); Assert.assertEquals( - new SegmentMetadataCache.FirstTypeMergePolicy(), + new AbstractSegmentMetadataCache.FirstTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy() ); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 78ba2add3627..503431341abf 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -63,7 +63,7 @@ public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - private SegmentMetadataCache runningSchema; + private SegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @@ -85,15 +85,15 @@ public void tearDown() throws Exception walker.close(); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException + public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException + public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException { Preconditions.checkState(runningSchema == null); - runningSchema = new SegmentMetadataCache( + runningSchema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, config, @@ -103,9 +103,9 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch( ) { @Override - public DataSourceInformation buildDruidTable(String dataSource) + public RowSignature buildDruidTable(String dataSource) { - DataSourceInformation table = super.buildDruidTable(dataSource); + RowSignature table = super.buildDruidTable(dataSource); buildTableLatch.countDown(); return table; } @@ -126,7 +126,7 @@ public void markDataSourceAsNeedRebuild(String datasource) @Test public void testGetTableMap() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); final Set tableNames = schema.getDatasourceNames(); @@ -136,7 +136,7 @@ public void testGetTableMap() throws InterruptedException @Test public void testGetTableMapFoo() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -164,7 +164,7 @@ public void testGetTableMapFoo() throws InterruptedException @Test public void testGetTableMapFoo2() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo2"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -185,12 +185,12 @@ public void testGetTableMapSomeTable() throws InterruptedException { // using 'newest first' column type merge strategy, the types are expected to be the types defined in the newer // segment, except for json, which is special handled - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( new SegmentMetadataCacheConfig() { @Override - public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + public AbstractSegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() { - return new SegmentMetadataCache.FirstTypeMergePolicy(); + return new AbstractSegmentMetadataCache.FirstTypeMergePolicy(); } } ); @@ -232,7 +232,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt { // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); final RowSignature fooRowSignature = fooDs.getRowSignature(); @@ -269,35 +269,35 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt /** * This tests that {@link AvailableSegmentMetadata#getNumRows()} is correct in case - * of multiple replicas i.e. when {@link SegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} + * of multiple replicas i.e. when {@link AbstractSegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} * is called more than once for same segment * @throws InterruptedException */ @Test public void testAvailableSegmentMetadataNumRows() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullAvailableSegmentMetadata(schema); } @Test public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() .stream() @@ -354,7 +354,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -395,7 +395,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -440,7 +440,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -482,7 +482,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -521,7 +521,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -577,7 +577,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -636,7 +636,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr { String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -669,7 +669,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -715,7 +715,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -785,7 +785,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. - SegmentMetadataCache mySchema = new SegmentMetadataCache( + SegmentMetadataCache mySchema = new SegmentMetadataCache( factoryMock, serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -817,7 +817,7 @@ public void testSegmentMetadataColumnType() new ColumnAnalysis(ColumnType.DOUBLE, ColumnType.DOUBLE.asTypeString(), false, true, 1234, 26, null, null, null) ); - RowSignature signature = SegmentMetadataCache.analysisToRowSignature( + RowSignature signature = AbstractSegmentMetadataCache.analysisToRowSignature( new SegmentAnalysis( "id", ImmutableList.of(Intervals.utc(1L, 2L)), @@ -844,7 +844,7 @@ public void testSegmentMetadataColumnType() @Test public void testSegmentMetadataFallbackType() { - RowSignature signature = SegmentMetadataCache.analysisToRowSignature( + RowSignature signature = AbstractSegmentMetadataCache.analysisToRowSignature( new SegmentAnalysis( "id", ImmutableList.of(Intervals.utc(1L, 2L)), @@ -905,7 +905,7 @@ public void testSegmentMetadataFallbackType() @Test public void testStaleDatasourceRefresh() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Set segments = new HashSet<>(); Set datasources = new HashSet<>(); datasources.add("wat"); @@ -920,7 +920,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index b111c1147cc8..200c3e71ce7c 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -74,7 +74,6 @@ import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory; import org.apache.druid.java.util.common.lifecycle.Lifecycle; import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.metadata.MetadataRuleManager; import org.apache.druid.metadata.MetadataRuleManagerConfig; @@ -88,11 +87,10 @@ import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; -import org.apache.druid.segment.metadata.DataSourceInformation; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.ClientQuerySegmentWalker; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.CoordinatorConfigManager; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -128,7 +126,6 @@ import org.apache.druid.server.lookup.cache.LookupCoordinatorManagerConfig; import org.apache.druid.server.metrics.ServiceStatusMonitor; import org.apache.druid.server.router.TieredBrokerConfig; -import org.apache.druid.server.security.Escalator; import org.eclipse.jetty.server.Server; import org.joda.time.Duration; @@ -228,7 +225,7 @@ public void configure(Binder binder) } else { binder.bind(CoordinatorTimeline.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); - binder.bind(SegmentMetadataCache.class).toProvider(Providers.of(null)); + binder.bind(AbstractSegmentMetadataCache.class).toProvider(Providers.of(null)); } binder.bind(SegmentsMetadataManager.class) @@ -484,48 +481,7 @@ public void configure(Binder binder) binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, QueryableCoordinatorServerView.class); - binder.bind(new TypeLiteral>() {}) - .toProvider(SegmentMetadataCacheProvider.class).in(ManageLifecycle.class); - } - - private static class SegmentMetadataCacheProvider implements Provider> - { - private final QueryLifecycleFactory queryLifecycleFactory; - private final TimelineServerView serverView; - private final SegmentMetadataCacheConfig config; - private final Escalator escalator; - private final InternalQueryConfig internalQueryConfig; - private final ServiceEmitter emitter; - - @Inject - public SegmentMetadataCacheProvider( - QueryLifecycleFactory queryLifecycleFactory, - TimelineServerView serverView, - SegmentMetadataCacheConfig config, - Escalator escalator, - InternalQueryConfig internalQueryConfig, - ServiceEmitter emitter - ) - { - this.queryLifecycleFactory = queryLifecycleFactory; - this.serverView = serverView; - this.config = config; - this.escalator = escalator; - this.internalQueryConfig = internalQueryConfig; - this.emitter = emitter; - } - - @Override - public SegmentMetadataCache get() - { - return new SegmentMetadataCache<>( - queryLifecycleFactory, - serverView, - config, - escalator, - internalQueryConfig, - emitter); - } + LifecycleModule.register(binder, SegmentMetadataCache.class); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 14803ea31417..e761ecd702d6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -29,8 +29,9 @@ import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; @@ -48,27 +49,19 @@ * metadata about a dataSource which is blended with catalog "logical" metadata * to provide the final user-view of each dataSource. *

    - * This class extends {@link SegmentMetadataCache} and introduces following changes, + * This class extends {@link AbstractSegmentMetadataCache} and introduces following changes, *

      *
    • The refresh mechanism now includes polling the coordinator for dataSource schema, * and falling back to running {@link org.apache.druid.query.metadata.metadata.SegmentMetadataQuery}.
      • - *
    • It builds and caches {@link PhysicalDatasourceMetadata} object as table - * schema instead of {@link DataSourceInformation}.
      • + *
    • It builds and caches {@link PhysicalDatasourceMetadata} object for the table schema
      • *
    */ @ManageLifecycle -public class BrokerSegmentMetadataCache extends SegmentMetadataCache +public class BrokerSegmentMetadataCache extends AbstractSegmentMetadataCache { private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataCache.class); private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; - - /** - * Manages tables of PhysicalDataSourceMetadata. This manager is used to retrieve and store - * information related to dataSources. - * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. - */ - //private final TableManager<> tableManager = new TableManager<>(); private final CoordinatorClient coordinatorClient; @Inject @@ -97,8 +90,10 @@ public BrokerSegmentMetadataCache( /** * Refreshes the set of segments in two steps: - * 1. Polls the coordinator for the dataSource schema to update the {@code tables}. - * 2. Refreshes the remaining set of segments by executing a SegmentMetadataQuery. + *
      + *
    • Polls the coordinator for the dataSource schema to update the {@code tables}.
      • + *
    • Refreshes the remaining set of segments by executing a SegmentMetadataQuery.
      • + *
    */ @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException @@ -112,13 +107,16 @@ public void refresh(final Set segmentsToRefresh, final Set da // Fetch dataSource information from the Coordinator try { FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) - .forEach(item -> polledDataSourceMetadata.put( - item.getDataSource(), - physicalDatasourceMetadataBuilder.build(item) + .forEach(dataSourceInformation -> polledDataSourceMetadata.put( + dataSourceInformation.getDataSource(), + physicalDatasourceMetadataBuilder.build( + dataSourceInformation.getDataSource(), + dataSourceInformation.getRowSignature() + ) )); } catch (Exception e) { - log.warn(e, "Exception querying coordinator to fetch dataSourceInformation."); + log.warn("Failed to query dataSource information from the Coordinator."); } // remove any extra dataSources returned @@ -147,16 +145,17 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - final DataSourceInformation druidTable = buildDruidTable(dataSource); - if (druidTable == null) { + final RowSignature rowSignature = buildDruidTable(dataSource); + if (rowSignature == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); + + final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(dataSource, rowSignature); final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); if (oldTable == null || !oldTable.getRowSignature().equals(physicalDatasourceMetadata.getRowSignature())) { - log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + log.info("[%s] has new signature: %s.", dataSource, rowSignature); } else { log.debug("[%s] signature is unchanged.", dataSource); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index c5922b066885..4ef9d63788e9 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -22,6 +22,7 @@ import com.google.inject.Inject; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.TableDataSource; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; @@ -46,9 +47,8 @@ public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, Segmen /** * Builds physical metadata for the given data source. */ - PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) + PhysicalDatasourceMetadata build(final String dataSource, final RowSignature rowSignature) { - final String dataSource = dataSourceInformation.getDataSource(); final TableDataSource tableDataSource; // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing @@ -66,7 +66,7 @@ PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) } return new PhysicalDatasourceMetadata( tableDataSource, - dataSourceInformation.getRowSignature(), + rowSignature, isJoinable, isBroadcast ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java index 25e6096bb190..a2f5ef4b5395 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java @@ -24,7 +24,7 @@ import org.apache.druid.guice.GuiceInjectors; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; -import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.sql.calcite.planner.CalcitePlannerModule; import org.joda.time.Period; import org.junit.Assert; @@ -52,7 +52,7 @@ public void testDefaultConfig() Assert.assertTrue(config.isAwaitInitializationOnStart()); Assert.assertTrue(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); - Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); + Assert.assertEquals(new AbstractSegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } @Test @@ -76,7 +76,7 @@ public void testCustomizedConfig() Assert.assertTrue(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); Assert.assertEquals( - new SegmentMetadataCache.FirstTypeMergePolicy(), + new AbstractSegmentMetadataCache.FirstTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy() ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index f4b55fb5929a..a913dfcc9a5b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -52,9 +52,9 @@ import org.apache.druid.segment.join.Joinable; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheCommon; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; @@ -105,7 +105,6 @@ public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon Set segmentDataSourceNames; Set joinableDataSourceNames; JoinableFactory globalTableJoinable; - CountDownLatch getDatasourcesLatch = new CountDownLatch(1); private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); @@ -122,7 +121,6 @@ public void setUp() throws Exception @Override public Set getDataSourceNames() { - getDatasourcesLatch.countDown(); return segmentDataSourceNames; } }; @@ -178,9 +176,9 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMeta ) { @Override - public DataSourceInformation buildDruidTable(String dataSource) + public RowSignature buildDruidTable(String dataSource) { - DataSourceInformation table = super.buildDruidTable(dataSource); + RowSignature table = super.buildDruidTable(dataSource); buildTableLatch.countDown(); return table; } @@ -410,9 +408,9 @@ public void testGetTableMapSomeTable() throws InterruptedException BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch( new BrokerSegmentMetadataCacheConfig() { @Override - public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + public AbstractSegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() { - return new SegmentMetadataCache.FirstTypeMergePolicy(); + return new AbstractSegmentMetadataCache.FirstTypeMergePolicy(); } }, new NoopCoordinatorClient() @@ -500,7 +498,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt /** * This tests that {@link AvailableSegmentMetadata#getNumRows()} is correct in case - * of multiple replicas i.e. when {@link SegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} + * of multiple replicas i.e. when {@link AbstractSegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} * is called more than once for same segment * @throws InterruptedException */ diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java index ccdbef25461b..3847bddd43ac 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -28,7 +28,6 @@ import org.apache.druid.segment.join.Joinable; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; -import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; import org.apache.druid.sql.calcite.table.DatasourceTable; import org.easymock.EasyMock; @@ -38,12 +37,9 @@ import java.util.Optional; import java.util.Set; -import java.util.concurrent.CountDownLatch; public class PhysicalDataSourceMetadataBuilderTest { - private CountDownLatch getDatasourcesLatch = new CountDownLatch(1); - private Set segmentDataSourceNames; private Set joinableDataSourceNames; private SegmentManager segmentManager; @@ -61,7 +57,6 @@ public void setUp() @Override public Set getDataSourceNames() { - getDatasourcesLatch.countDown(); return segmentDataSourceNames; } }; @@ -94,34 +89,28 @@ public void testBuild() segmentDataSourceNames.add("foo"); joinableDataSourceNames.add("foo"); - DataSourceInformation foo = - new DataSourceInformation( - "foo", - RowSignature.builder() - .add("c1", ColumnType.FLOAT) - .add("c2", ColumnType.DOUBLE) - .build() - ); + RowSignature fooSignature = + RowSignature.builder() + .add("c1", ColumnType.FLOAT) + .add("c2", ColumnType.DOUBLE) + .build(); - DataSourceInformation bar = - new DataSourceInformation( - "bar", - RowSignature.builder() - .add("d1", ColumnType.FLOAT) - .add("d2", ColumnType.DOUBLE) - .build() - ); + RowSignature barSignature = + RowSignature.builder() + .add("d1", ColumnType.FLOAT) + .add("d2", ColumnType.DOUBLE) + .build(); - DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build(foo); + DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build("foo", fooSignature); Assert.assertTrue(fooDs.isJoinable()); Assert.assertTrue(fooDs.isBroadcast()); - Assert.assertEquals(fooDs.dataSource().getName(), foo.getDataSource()); - Assert.assertEquals(fooDs.getRowSignature(), foo.getRowSignature()); + Assert.assertEquals(fooDs.dataSource().getName(), "foo"); + Assert.assertEquals(fooDs.getRowSignature(), fooSignature); - DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build(bar); + DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build("bar", barSignature); Assert.assertFalse(barDs.isJoinable()); Assert.assertFalse(barDs.isBroadcast()); - Assert.assertEquals(barDs.dataSource().getName(), bar.getDataSource()); - Assert.assertEquals(barDs.getRowSignature(), bar.getRowSignature()); + Assert.assertEquals(barDs.dataSource().getName(), "bar"); + Assert.assertEquals(barDs.getRowSignature(), barSignature); } } From 7badce1612f48ffbf9e41f38417b1224d0a6323b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 00:19:58 +0530 Subject: [PATCH 34/82] Rename SegmentMetadataCache to CoordinatorSegmentMetadataCache --- ...ruidSchemaInternRowSignatureBenchmark.java | 4 +- ...a => CoordinatorSegmentMetadataCache.java} | 4 +- .../druid/server/http/MetadataResource.java | 20 +++---- ...inatorSegmentMetadataCacheConfigTest.java} | 2 +- ... CoordinatorSegmentMetadataCacheTest.java} | 54 +++++++++---------- .../SegmentDataCacheConcurrencyTest.java | 4 +- .../server/http/MetadataResourceTest.java | 18 +++---- .../org/apache/druid/cli/CliCoordinator.java | 4 +- .../schema/BrokerSegmentMetadataCache.java | 1 - .../PhysicalDatasourceMetadataBuilder.java | 1 - .../BrokerSegmentMetadataCacheTest.java | 6 +-- 11 files changed, 57 insertions(+), 61 deletions(-) rename server/src/main/java/org/apache/druid/segment/metadata/{SegmentMetadataCache.java => CoordinatorSegmentMetadataCache.java} (96%) rename server/src/test/java/org/apache/druid/segment/metadata/{SegmentMetadataCacheConfigTest.java => CoordinatorSegmentMetadataCacheConfigTest.java} (98%) rename server/src/test/java/org/apache/druid/segment/metadata/{SegmentMetadataCacheTest.java => CoordinatorSegmentMetadataCacheTest.java} (94%) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 5f0634f283a4..85a9931ab037 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -30,7 +30,7 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -68,7 +68,7 @@ public class DruidSchemaInternRowSignatureBenchmark { private SegmentMetadataCacheForBenchmark cache; - private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache + private static class SegmentMetadataCacheForBenchmark extends CoordinatorSegmentMetadataCache { public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java similarity index 96% rename from server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java rename to server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index 83c67532d6e2..a24308ff4723 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -39,12 +39,12 @@ * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle -public class SegmentMetadataCache extends AbstractSegmentMetadataCache +public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCache { private static final EmittingLogger log = new EmittingLogger(AbstractSegmentMetadataCache.class); @Inject - public SegmentMetadataCache( + public CoordinatorSegmentMetadataCache( QueryLifecycleFactory queryLifecycleFactory, TimelineServerView serverView, SegmentMetadataCacheConfig config, diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index c79962c0d111..9a8fb176acfa 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -30,8 +30,8 @@ import org.apache.druid.indexing.overlord.Segments; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.JettyUtils; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.http.security.DatasourceResourceFilter; @@ -75,7 +75,7 @@ public class MetadataResource private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; - private final @Nullable SegmentMetadataCache segmentMetadataCache; + private final @Nullable CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache; @Inject public MetadataResource( @@ -83,14 +83,14 @@ public MetadataResource( IndexerMetadataStorageCoordinator metadataStorageCoordinator, AuthorizerMapper authorizerMapper, DruidCoordinator coordinator, - @Nullable SegmentMetadataCache segmentMetadataCache + @Nullable CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache ) { this.segmentsMetadataManager = segmentsMetadataManager; this.metadataStorageCoordinator = metadataStorageCoordinator; this.authorizerMapper = authorizerMapper; this.coordinator = coordinator; - this.segmentMetadataCache = segmentMetadataCache; + this.coordinatorSegmentMetadataCache = coordinatorSegmentMetadataCache; } @GET @@ -203,8 +203,8 @@ private Response getAllUsedSegmentsWithAdditionalDetails( : coordinator.getReplicationFactor(segment.getId()); Long numRows = null; - if (null != segmentMetadataCache) { - AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( + if (null != coordinatorSegmentMetadataCache) { + AvailableSegmentMetadata availableSegmentMetadata = coordinatorSegmentMetadataCache.getAvailableSegmentMetadata( segment.getDataSource(), segment.getId() ); @@ -226,8 +226,8 @@ private Response getAllUsedSegmentsWithAdditionalDetails( Stream finalSegments = segmentStatus; // conditionally add realtime segments information - if (null != includeRealtimeSegments && null != segmentMetadataCache) { - final Stream realtimeSegmentStatus = segmentMetadataCache + if (null != includeRealtimeSegments && null != coordinatorSegmentMetadataCache) { + final Stream realtimeSegmentStatus = coordinatorSegmentMetadataCache .getSegmentMetadataSnapshot() .values() .stream() @@ -366,10 +366,10 @@ public Response getDataSourceInformation( List dataSources ) { - if (null == segmentMetadataCache) { + if (null == coordinatorSegmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } - Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceInformationMap(); + Map dataSourceSchemaMap = coordinatorSegmentMetadataCache.getDataSourceInformationMap(); List results = new ArrayList<>(); List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheConfigTest.java similarity index 98% rename from server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheConfigTest.java index 119191dc767f..9668466396e0 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheConfigTest.java @@ -33,7 +33,7 @@ /** * Pathetic little unit test just to keep Jacoco happy. */ -public class SegmentMetadataCacheConfigTest +public class CoordinatorSegmentMetadataCacheConfigTest { private static final String CONFIG_BASE = "druid.coordinator.segmentMetadataCache"; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java similarity index 94% rename from server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java index 503431341abf..90664e7c87d0 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java @@ -58,12 +58,12 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon +public class CoordinatorSegmentMetadataCacheTest extends SegmentMetadataCacheCommon { // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); - static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - private SegmentMetadataCache runningSchema; + private static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); + private CoordinatorSegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @@ -85,15 +85,15 @@ public void tearDown() throws Exception walker.close(); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException + public CoordinatorSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException + public CoordinatorSegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException { Preconditions.checkState(runningSchema == null); - runningSchema = new SegmentMetadataCache( + runningSchema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, config, @@ -126,7 +126,7 @@ public void markDataSourceAsNeedRebuild(String datasource) @Test public void testGetTableMap() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); final Set tableNames = schema.getDatasourceNames(); @@ -136,7 +136,7 @@ public void testGetTableMap() throws InterruptedException @Test public void testGetTableMapFoo() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -164,7 +164,7 @@ public void testGetTableMapFoo() throws InterruptedException @Test public void testGetTableMapFoo2() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo2"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -185,7 +185,7 @@ public void testGetTableMapSomeTable() throws InterruptedException { // using 'newest first' column type merge strategy, the types are expected to be the types defined in the newer // segment, except for json, which is special handled - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch( new SegmentMetadataCacheConfig() { @Override public AbstractSegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() @@ -232,7 +232,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt { // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); final RowSignature fooRowSignature = fooDs.getRowSignature(); @@ -276,28 +276,28 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt @Test public void testAvailableSegmentMetadataNumRows() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullAvailableSegmentMetadata(schema); } @Test public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() .stream() @@ -354,7 +354,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -395,7 +395,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -440,7 +440,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -482,7 +482,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -521,7 +521,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -577,7 +577,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -636,7 +636,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr { String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -669,7 +669,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -715,7 +715,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -785,7 +785,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. - SegmentMetadataCache mySchema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache mySchema = new CoordinatorSegmentMetadataCache( factoryMock, serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -905,7 +905,7 @@ public void testSegmentMetadataFallbackType() @Test public void testStaleDatasourceRefresh() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Set segments = new HashSet<>(); Set datasources = new HashSet<>(); datasources.add("wat"); @@ -920,7 +920,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 1c9256da0fc8..a26b82722f86 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -131,7 +131,7 @@ public void tearDown() throws Exception public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerViewGetTimeline() throws InterruptedException, ExecutionException, TimeoutException { - schema = new SegmentMetadataCache( + schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -239,7 +239,7 @@ public CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegme public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() throws InterruptedException, ExecutionException, TimeoutException { - schema = new SegmentMetadataCache( + schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index e122334946e5..a430081c10a7 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -32,8 +32,8 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.security.AuthConfig; @@ -140,7 +140,7 @@ public void testGetAllSegmentsWithOvershadowedStatus() @Test public void testGetAllSegmentsIncludingRealtime() { - SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); + CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache = Mockito.mock(CoordinatorSegmentMetadataCache.class); String dataSource2 = "datasource2"; @@ -205,14 +205,14 @@ public void testGetAllSegmentsIncludingRealtime() ).build() ); - Mockito.doReturn(availableSegments).when(segmentMetadataCache).getSegmentMetadataSnapshot(); + Mockito.doReturn(availableSegments).when(coordinatorSegmentMetadataCache).getSegmentMetadataSnapshot(); Mockito.doReturn(availableSegments.get(segments[0].getId())) - .when(segmentMetadataCache) + .when(coordinatorSegmentMetadataCache) .getAvailableSegmentMetadata(DATASOURCE1, segments[0].getId()); Mockito.doReturn(availableSegments.get(segments[1].getId())) - .when(segmentMetadataCache) + .when(coordinatorSegmentMetadataCache) .getAvailableSegmentMetadata(DATASOURCE1, segments[1].getId()); metadataResource = new MetadataResource( @@ -220,7 +220,7 @@ public void testGetAllSegmentsIncludingRealtime() storageCoordinator, AuthTestUtils.TEST_AUTHORIZER_MAPPER, coordinator, - segmentMetadataCache + coordinatorSegmentMetadataCache ); Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", "includeRealtimeSegments"); @@ -239,7 +239,7 @@ public void testGetAllSegmentsIncludingRealtime() @Test public void testGetDataSourceInformation() { - SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); + CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache = Mockito.mock(CoordinatorSegmentMetadataCache.class); Map dataSourceInformationMap = new HashMap<>(); dataSourceInformationMap.put( @@ -264,14 +264,14 @@ public void testGetDataSourceInformation() ) ); - Mockito.doReturn(dataSourceInformationMap).when(segmentMetadataCache).getDataSourceInformationMap(); + Mockito.doReturn(dataSourceInformationMap).when(coordinatorSegmentMetadataCache).getDataSourceInformationMap(); metadataResource = new MetadataResource( segmentsMetadataManager, storageCoordinator, AuthTestUtils.TEST_AUTHORIZER_MAPPER, coordinator, - segmentMetadataCache + coordinatorSegmentMetadataCache ); Response response = metadataResource.getDataSourceInformation(Collections.singletonList(DATASOURCE1)); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 200c3e71ce7c..c042e9de2409 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -88,7 +88,7 @@ import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; -import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.ClientQuerySegmentWalker; import org.apache.druid.server.audit.AuditManagerProvider; @@ -481,7 +481,7 @@ public void configure(Binder binder) binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, QueryableCoordinatorServerView.class); - LifecycleModule.register(binder, SegmentMetadataCache.class); + LifecycleModule.register(binder, CoordinatorSegmentMetadataCache.class); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index e761ecd702d6..9c5e9b5f5e24 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -31,7 +31,6 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; -import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index 4ef9d63788e9..37a106bdd9ba 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -24,7 +24,6 @@ import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; -import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index a913dfcc9a5b..40a254f9439b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -93,10 +93,10 @@ public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon { - private final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); + private static final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final int WAIT_TIMEOUT_SECS = 6; - + private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); private BrokerSegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @@ -106,8 +106,6 @@ public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon Set joinableDataSourceNames; JoinableFactory globalTableJoinable; - private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); - @Before public void setUp() throws Exception { From 25cdce6bec144332d11e8ecedfea86700abb5f99 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 00:32:03 +0530 Subject: [PATCH 35/82] Rename PhysicalDataSourceMetadataBuilder to PhysicalDataSourceMetadataFactory --- .../calcite/schema/BrokerSegmentMetadataCache.java | 10 +++++----- ...r.java => PhysicalDatasourceMetadataFactory.java} | 4 ++-- .../schema/BrokerSegmentMetadataCacheTest.java | 12 ++++++------ .../calcite/schema/DruidSchemaNoDataInitTest.java | 2 +- ...va => PhysicalDataSourceMetadataFactoryTest.java} | 10 +++++----- .../druid/sql/calcite/schema/SystemSchemaTest.java | 2 +- .../druid/sql/calcite/util/QueryFrameworkUtils.java | 4 ++-- 7 files changed, 22 insertions(+), 22 deletions(-) rename sql/src/main/java/org/apache/druid/sql/calcite/schema/{PhysicalDatasourceMetadataBuilder.java => PhysicalDatasourceMetadataFactory.java} (96%) rename sql/src/test/java/org/apache/druid/sql/calcite/schema/{PhysicalDataSourceMetadataBuilderTest.java => PhysicalDataSourceMetadataFactoryTest.java} (88%) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 9c5e9b5f5e24..7113e3fbce81 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -60,7 +60,7 @@ public class BrokerSegmentMetadataCache extends AbstractSegmentMetadataCache segmentsToRefresh, final Set da FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) .forEach(dataSourceInformation -> polledDataSourceMetadata.put( dataSourceInformation.getDataSource(), - physicalDatasourceMetadataBuilder.build( + dataSourceMetadataFactory.build( dataSourceInformation.getDataSource(), dataSourceInformation.getRowSignature() ) @@ -151,7 +151,7 @@ public void refresh(final Set segmentsToRefresh, final Set da return; } - final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(dataSource, rowSignature); + final PhysicalDatasourceMetadata physicalDatasourceMetadata = dataSourceMetadataFactory.build(dataSource, rowSignature); final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); if (oldTable == null || !oldTable.getRowSignature().equals(physicalDatasourceMetadata.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, rowSignature); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java similarity index 96% rename from sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java rename to sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java index 37a106bdd9ba..106a0d401629 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java @@ -31,13 +31,13 @@ * Builds {@link PhysicalDatasourceMetadata} for a dataSource, including information about its schema, * joinability, and broadcast status. */ -public class PhysicalDatasourceMetadataBuilder +public class PhysicalDatasourceMetadataFactory { private final JoinableFactory joinableFactory; private final SegmentManager segmentManager; @Inject - public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, SegmentManager segmentManager) + public PhysicalDatasourceMetadataFactory(JoinableFactory joinableFactory, SegmentManager segmentManager) { this.joinableFactory = joinableFactory; this.segmentManager = segmentManager; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 40a254f9439b..ad804b78e918 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -169,7 +169,7 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMeta new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), coordinatorClient ) { @@ -204,7 +204,7 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws Interr new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), new NoopCoordinatorClient() ) { @@ -265,7 +265,7 @@ public ListenableFuture> fetchDataSourceInformation( new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), coordinatorClient ); @@ -328,7 +328,7 @@ public ListenableFuture> fetchDataSourceInformation( new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), coordinatorClient ); @@ -693,7 +693,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception new NoopEscalator(), internalQueryConfig, new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), new NoopCoordinatorClient() ); @@ -725,7 +725,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept new NoopEscalator(), new InternalQueryConfig(), emitter, - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), new NoopCoordinatorClient() ) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index 2ac84ed76de1..754136d853d4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -59,7 +59,7 @@ public void testInitializationWithNoData() throws Exception new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder( + new PhysicalDatasourceMetadataFactory( new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), new SegmentManager(EasyMock.createMock(SegmentLoader.class))), null diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataFactoryTest.java similarity index 88% rename from sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java rename to sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataFactoryTest.java index 3847bddd43ac..4700a387d0e3 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataFactoryTest.java @@ -38,14 +38,14 @@ import java.util.Optional; import java.util.Set; -public class PhysicalDataSourceMetadataBuilderTest +public class PhysicalDataSourceMetadataFactoryTest { private Set segmentDataSourceNames; private Set joinableDataSourceNames; private SegmentManager segmentManager; private JoinableFactory globalTableJoinable; - private PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + private PhysicalDatasourceMetadataFactory datasourceMetadataFactory; @Before public void setUp() @@ -80,7 +80,7 @@ public Optional build( } }; - physicalDatasourceMetadataBuilder = new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager); + datasourceMetadataFactory = new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager); } @Test @@ -101,13 +101,13 @@ public void testBuild() .add("d2", ColumnType.DOUBLE) .build(); - DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build("foo", fooSignature); + DatasourceTable.PhysicalDatasourceMetadata fooDs = datasourceMetadataFactory.build("foo", fooSignature); Assert.assertTrue(fooDs.isJoinable()); Assert.assertTrue(fooDs.isBroadcast()); Assert.assertEquals(fooDs.dataSource().getName(), "foo"); Assert.assertEquals(fooDs.getRowSignature(), fooSignature); - DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build("bar", barSignature); + DatasourceTable.PhysicalDatasourceMetadata barDs = datasourceMetadataFactory.build("bar", barSignature); Assert.assertFalse(barDs.isJoinable()); Assert.assertFalse(barDs.isBroadcast()); Assert.assertEquals(barDs.dataSource().getName(), "bar"); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 4f0aaaa8e9df..46f6c5fda9b2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -258,7 +258,7 @@ public void setUp() throws Exception new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder( + new PhysicalDatasourceMetadataFactory( new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), new SegmentManager(EasyMock.createMock(SegmentLoader.class)) ), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 14e1912a93a9..0592d7de8afa 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -71,7 +71,7 @@ import org.apache.druid.sql.calcite.schema.NamedSystemSchema; import org.apache.druid.sql.calcite.schema.NamedViewSchema; import org.apache.druid.sql.calcite.schema.NoopDruidSchemaManager; -import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataBuilder; +import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataFactory; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.sql.calcite.schema.ViewSchema; import org.apache.druid.sql.calcite.view.ViewManager; @@ -218,7 +218,7 @@ private static DruidSchema createMockSchema( CalciteTests.TEST_AUTHENTICATOR_ESCALATOR, new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder( + new PhysicalDatasourceMetadataFactory( createDefaultJoinableFactory(injector), new SegmentManager(EasyMock.createMock(SegmentLoader.class)) { From 5f5ad18202d4635d890693ee637c963325edf257 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 01:58:29 +0530 Subject: [PATCH 36/82] Fix json property key name in DataSourceInformation --- .../client/QueryableCoordinatorServerView.java | 5 ++--- .../CoordinatorSegmentMetadataCache.java | 2 +- .../segment/metadata/DataSourceInformation.java | 2 +- .../org/apache/druid/cli/CliCoordinator.java | 17 +++++++++-------- .../schema/BrokerSegmentMetadataCache.java | 7 ++++--- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 1db03afa2f85..96d5d80eee72 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -32,7 +32,6 @@ import org.apache.druid.query.DataSource; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; -import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.VersionedIntervalTimeline; @@ -50,7 +49,7 @@ * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline.

    * *

    A new timeline class (implementing {@link TimelineServerView}) is required for - * {@link AbstractSegmentMetadataCache}, which will run on the Coordinator.

    + * {@link org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache}, which will run on the Coordinator.

    */ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline @@ -81,7 +80,7 @@ public boolean isAwaitInitializationOnStart() /** * This class maintains a timeline of {@link ServerSelector} objects. - * This method returns a new timeline of the object {@link SegmentLoadInfo}. + * This method converts and returns a new timeline of the object {@link SegmentLoadInfo}. * * @param dataSource dataSoruce * @return timeline for the given dataSource diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index a24308ff4723..15bb1f7c1fab 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -41,7 +41,7 @@ @ManageLifecycle public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCache { - private static final EmittingLogger log = new EmittingLogger(AbstractSegmentMetadataCache.class); + private static final EmittingLogger log = new EmittingLogger(CoordinatorSegmentMetadataCache.class); @Inject public CoordinatorSegmentMetadataCache( diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index f25c104203c5..e007fdb9af18 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -36,7 +36,7 @@ public class DataSourceInformation @JsonCreator public DataSourceInformation( - @JsonProperty("datasource") String dataSource, + @JsonProperty("dataSource") String dataSource, @JsonProperty("rowSignature") RowSignature rowSignature) { this.dataSource = Preconditions.checkNotNull(dataSource, "'dataSource' must be nonnull"); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index c042e9de2409..d60bc67e4ce1 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -87,7 +87,6 @@ import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; -import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.ClientQuerySegmentWalker; @@ -221,11 +220,11 @@ public void configure(Binder binder) } if (isSegmentMetadataCacheEnabled()) { - binder.install(new SegmentMetadataCacheModule()); + binder.install(new CoordinatorSegmentMetadataCacheModule()); } else { binder.bind(CoordinatorTimeline.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); - binder.bind(AbstractSegmentMetadataCache.class).toProvider(Providers.of(null)); + binder.bind(CoordinatorSegmentMetadataCache.class).toProvider(Providers.of(null)); } binder.bind(SegmentsMetadataManager.class) @@ -457,11 +456,13 @@ public Supplier> get() } } - private static class SegmentMetadataCacheModule implements Module + private static class CoordinatorSegmentMetadataCacheModule implements Module { @Override public void configure(Binder binder) { + // These modules are required to allow running queries on the Coordinator, + // since CoordinatorSegmentMetadataCache needs to query data nodes and tasks binder.install(new LegacyBrokerParallelMergeConfigModule()); binder.install(new QueryRunnerFactoryModule()); binder.install(new SegmentWranglerModule()); @@ -470,10 +471,10 @@ public void configure(Binder binder) binder.install(new JoinableFactoryModule()); JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.select", TierSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.coordinator.select.tier.custom", CustomTierSelectorStrategyConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.balancer", ServerSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.coordinator.retryPolicy", RetryQueryRunnerConfig.class); binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); binder.bind(CachingClusteredClient.class).in(LazySingleton.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 7113e3fbce81..42716f436b81 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -19,9 +19,9 @@ package org.apache.druid.sql.calcite.schema; +import com.google.common.base.Predicates; import com.google.common.collect.Sets; import com.google.inject.Inject; -import io.vavr.Predicates; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.CoordinatorClient; @@ -90,8 +90,9 @@ public BrokerSegmentMetadataCache( /** * Refreshes the set of segments in two steps: *
      - *
    • Polls the coordinator for the dataSource schema to update the {@code tables}.
      • - *
    • Refreshes the remaining set of segments by executing a SegmentMetadataQuery.
      • + *
    • Polls the coordinator for the dataSource schema.
      • + *
    • Refreshes the remaining set of segments by executing a SegmentMetadataQuery and + * builds dataSource schema by combining segment schema.
      • *
    */ @Override From 08e949ec69924edeb543b01d9fc66ef5099f5b9c Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 15:38:26 +0530 Subject: [PATCH 37/82] Add validation in MetadataResource#getAllUsedSegments, update javadocs --- .../QueryableCoordinatorServerView.java | 4 +- .../AbstractSegmentMetadataCache.java | 105 +++++++++++------- .../metadata/SegmentMetadataCacheConfig.java | 11 ++ .../druid/server/http/MetadataResource.java | 9 ++ .../schema/BrokerSegmentMetadataCache.java | 4 +- .../BrokerSegmentMetadataCacheConfig.java | 13 ++- 6 files changed, 100 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 96d5d80eee72..68073ade2027 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -79,8 +79,8 @@ public boolean isAwaitInitializationOnStart() } /** - * This class maintains a timeline of {@link ServerSelector} objects. - * This method converts and returns a new timeline of the object {@link SegmentLoadInfo}. + * Since this class maintains a timeline of {@link ServerSelector} objects, + * this method converts and returns a new timeline of the object {@link SegmentLoadInfo}. * * @param dataSource dataSoruce * @return timeline for the given dataSource diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index aefc16b679b3..657e99d44025 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -89,13 +89,13 @@ import java.util.stream.StreamSupport; /** - * An abstract class that listens for segment change events and caches segment metadata. - * It periodically queries data nodes to fetch segment schemas and combines them to build a data source schema. + * An abstract class that listens for segment change events and caches segment metadata and periodically refreshes + * the segments and dataSources. * *

    This class is generic and is parameterized by a type {@code T} that extends {@link DataSourceInformation}.

    * *

    This class has an abstract method {@link #refresh(Set, Set)} which the child class must override - * with the logic to get segment schema.

    + * with the logic to build and cache table schema.

    * * @param The type of information associated with the data source, which must extend {@link DataSourceInformation}. */ @@ -121,12 +121,6 @@ public abstract class AbstractSegmentMetadataCache tables = new ConcurrentHashMap<>(); - /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. * Use SortedMap for segments so they are merged in deterministic order, from older to newer. @@ -197,6 +191,12 @@ public abstract class AbstractSegmentMetadataCache tables = new ConcurrentHashMap<>(); + /** * This lock coordinates the access from multiple threads to those variables guarded by this lock. * Currently, there are 2 threads that can access these variables. @@ -395,8 +395,6 @@ public void start() throws InterruptedException } } - public abstract void refresh(Set segmentsToRefresh, Set dataSourcesToRebuild) throws IOException; - @LifecycleStop public void stop() { @@ -409,21 +407,79 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } + /** + * @param name name of the dataSource. + * @return schema information for the dataSource. + */ public T getDatasource(String name) { return tables.get(name); } + /** + * @return Map of dataSource and corresponding schema information. + */ public Map getDataSourceInformationMap() { return ImmutableMap.copyOf(tables); } + /** + * @return set of dataSources for which schema information is cached. + */ public Set getDatasourceNames() { return tables.keySet(); } + /** + * Get metadata for all the cached segments, which includes information like RowSignature, realtime & numRows etc. + * + * @return Map of segmentId and corresponding metadata. + */ + public Map getSegmentMetadataSnapshot() + { + final Map segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments); + for (ConcurrentSkipListMap val : segmentMetadataInfo.values()) { + segmentMetadata.putAll(val); + } + return segmentMetadata; + } + + /** + * Get metadata for the specified segment, which includes information like RowSignature, realtime & numRows. + * + * @param datasource dataSource of the given segment + * @param segmentId segmentId of the given segment + * @return Metadata information for the given segment + */ + @Nullable + public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId) + { + if (!segmentMetadataInfo.containsKey(datasource)) { + return null; + } + return segmentMetadataInfo.get(datasource).get(segmentId); + } + + /** + * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. + * As a result, the returned value might be inexact. + */ + public int getTotalSegments() + { + return totalSegments; + } + + /** + * The child classes must override this method with the logic to build and cache table schema. + * + * @param segmentsToRefresh suspected segments for which the schema might have changed + * @param dataSourcesToRebuild suspected dataSources for which the schema might have changed + * @throws IOException when querying segment from data nodes and tasks + */ + public abstract void refresh(Set segmentsToRefresh, Set dataSourcesToRebuild) throws IOException; + @VisibleForTesting public void addSegment(final DruidServerMetadata server, final DataSegment segment) { @@ -804,33 +860,6 @@ public RowSignature buildDruidTable(final String dataSource) return builder.build(); } - public Map getSegmentMetadataSnapshot() - { - final Map segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments); - for (ConcurrentSkipListMap val : segmentMetadataInfo.values()) { - segmentMetadata.putAll(val); - } - return segmentMetadata; - } - - @Nullable - public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId) - { - if (!segmentMetadataInfo.containsKey(datasource)) { - return null; - } - return segmentMetadataInfo.get(datasource).get(segmentId); - } - - /** - * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. - * As a result, the returned value might be inexact. - */ - public int getTotalSegments() - { - return totalSegments; - } - @VisibleForTesting public TreeSet getSegmentsNeedingRefresh() { diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 5857619e3947..656721b1b3dd 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -25,6 +25,7 @@ /** * Coordinator-side configuration class for customizing properties related to the SegmentMetadata cache. + * See {@link CoordinatorSegmentMetadataCache} */ public class SegmentMetadataCacheConfig { @@ -74,4 +75,14 @@ public Period getMetadataRefreshPeriod() { return metadataRefreshPeriod; } + + @Override + public String toString() + { + return "SegmentMetadataCacheConfig{" + + "awaitInitializationOnStart=" + awaitInitializationOnStart + + ", metadataRefreshPeriod=" + metadataRefreshPeriod + + ", metadataColumnTypeMergePolicy=" + metadataColumnTypeMergePolicy + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 9a8fb176acfa..b6f99e614337 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -150,6 +150,11 @@ public Response getAllUsedSegments( @QueryParam("includeRealtimeSegments") final @Nullable String includeRealtimeSegments ) { + // realtime segments can be requested only when {@code includeOverShadowedStatus} is set + if (includeOvershadowedStatus == null && includeRealtimeSegments != null) { + return Response.status(Response.Status.BAD_REQUEST).build(); + } + if (includeOvershadowedStatus != null) { return getAllUsedSegmentsWithAdditionalDetails(req, dataSources, includeRealtimeSegments); } @@ -358,6 +363,9 @@ public Response getSegment( /** * API to fetch {@link DataSourceInformation} for the specified dataSources. + * + * @param dataSources list of dataSources to be queried + * @return information including schema details for the specified dataSources */ @POST @Path("/dataSourceInformation") @@ -366,6 +374,7 @@ public Response getDataSourceInformation( List dataSources ) { + // if {@code coordinatorSegmentMetadataCache} is null, implies the feature is disabled. Return NOT_FOUND. if (null == coordinatorSegmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 42716f436b81..3a7d99d4bb25 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -98,11 +98,11 @@ public BrokerSegmentMetadataCache( @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { - Set dataSourcesToQuery = new HashSet<>(); + final Set dataSourcesToQuery = new HashSet<>(dataSourcesToRebuild); segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); - Map polledDataSourceMetadata = new HashMap<>(); + final Map polledDataSourceMetadata = new HashMap<>(); // Fetch dataSource information from the Coordinator try { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index fc6d94da2eb7..69e61440ff3b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -25,11 +25,16 @@ /** * Broker-side configuration class for managing segment polling from the Coordinator and - * customizing properties related to the SegmentMetadata cache. - *

    - * The property {@link #awaitInitializationOnStart} is overridden in this class with a default value + * customizing properties related to the SegmentMetadata cache which is used to infer datasources for SQL. + * + *

    See {@link BrokerSegmentMetadataCache}, {@link MetadataSegmentView}.

    + * + *

    This class shares the same config root as {@link org.apache.druid.sql.calcite.planner.PlannerConfig} + * to maintain backward compatibility for when the properties here resided in {@code PlannerConfig}.

    + * + *

    The property {@link #awaitInitializationOnStart} is overridden in this class with a default value * of {@code true}, which differs from the parent class. This ensures that the SegmentMetadata cache is - * fully initialized before other startup processes proceed. + * fully initialized before other startup processes proceed.

    */ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig { From 80fc09d83b29b0535ba339ed1b4adf4d7973d755 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 15:59:51 +0530 Subject: [PATCH 38/82] Minor changes --- .../AbstractSegmentMetadataCache.java | 20 +++++++++++-------- .../CoordinatorSegmentMetadataCache.java | 4 ++++ .../schema/BrokerSegmentMetadataCache.java | 4 ++++ .../BrokerSegmentMetadataCacheConfig.java | 12 +++++++++++ .../PhysicalDatasourceMetadataFactory.java | 5 +++++ 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index 657e99d44025..7cd3651533a6 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -408,8 +408,11 @@ public void awaitInitialization() throws InterruptedException } /** - * @param name name of the dataSource. - * @return schema information for the dataSource. + * Fetch schema for the given dataSource. + * + * @param name dataSource + * + * @return schema information for the given dataSource */ public T getDatasource(String name) { @@ -425,7 +428,7 @@ public Map getDataSourceInformationMap() } /** - * @return set of dataSources for which schema information is cached. + * @return Set of dataSources for which schema information is cached. */ public Set getDatasourceNames() { @@ -449,8 +452,9 @@ public Map getSegmentMetadataSnapshot() /** * Get metadata for the specified segment, which includes information like RowSignature, realtime & numRows. * - * @param datasource dataSource of the given segment - * @param segmentId segmentId of the given segment + * @param datasource segment dataSource + * @param segmentId segment Id + * * @return Metadata information for the given segment */ @Nullable @@ -474,9 +478,9 @@ public int getTotalSegments() /** * The child classes must override this method with the logic to build and cache table schema. * - * @param segmentsToRefresh suspected segments for which the schema might have changed - * @param dataSourcesToRebuild suspected dataSources for which the schema might have changed - * @throws IOException when querying segment from data nodes and tasks + * @param segmentsToRefresh segments for which the schema might have changed + * @param dataSourcesToRebuild dataSources for which the schema might have changed + * @throws IOException when querying segment schema from data nodes and tasks */ public abstract void refresh(Set segmentsToRefresh, Set dataSourcesToRebuild) throws IOException; diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index 15bb1f7c1fab..be86fcac0a9f 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -59,6 +59,10 @@ public CoordinatorSegmentMetadataCache( /** * Fires SegmentMetadataQuery to fetch schema information for each segment in the refresh list. * The schema information for individual segments is combined to construct a table schema, which is then cached. + * + * @param segmentsToRefresh suspected segments for which the schema might have changed + * @param dataSourcesToRebuild suspected dataSources for which the schema might have changed + * @throws IOException when querying segment from data nodes and tasks */ @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 3a7d99d4bb25..63f5b8a91b72 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -94,6 +94,10 @@ public BrokerSegmentMetadataCache( *
  • Refreshes the remaining set of segments by executing a SegmentMetadataQuery and * builds dataSource schema by combining segment schema.
    • * + * + * @param segmentsToRefresh segments for which the schema might have changed + * @param dataSourcesToRebuild dataSources for which the schema might have changed + * @throws IOException when querying segment schema from data nodes and tasks */ @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 69e61440ff3b..8aa70ac0c21b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -82,4 +82,16 @@ public boolean isAwaitInitializationOnStart() { return awaitInitializationOnStart; } + + @Override + public String toString() + { + return "BrokerSegmentMetadataCacheConfig{" + + "metadataSegmentCacheEnable=" + metadataSegmentCacheEnable + + ", metadataSegmentPollPeriod=" + metadataSegmentPollPeriod + + ", awaitInitializationOnStart=" + awaitInitializationOnStart + + ", metadataRefreshPeriod=" + getMetadataRefreshPeriod() + + ", metadataColumnTypeMergePolicy=" + getMetadataColumnTypeMergePolicy() + + '}'; + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java index 106a0d401629..65138c2f686f 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java @@ -45,6 +45,11 @@ public PhysicalDatasourceMetadataFactory(JoinableFactory joinableFactory, Segmen /** * Builds physical metadata for the given data source. + * + * @param dataSource name of the dataSource + * @param rowSignature schema of the dataSource + * + * @return PhysicalDatasourceMetadata which includes information about schema, joinability and broadcast status */ PhysicalDatasourceMetadata build(final String dataSource, final RowSignature rowSignature) { From 4217cd8bbe054d899187264930fd734736bfd9ae Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 16:00:59 +0530 Subject: [PATCH 39/82] Minor change --- .../segment/metadata/CoordinatorSegmentMetadataCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index be86fcac0a9f..a268c25c4790 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -60,8 +60,8 @@ public CoordinatorSegmentMetadataCache( * Fires SegmentMetadataQuery to fetch schema information for each segment in the refresh list. * The schema information for individual segments is combined to construct a table schema, which is then cached. * - * @param segmentsToRefresh suspected segments for which the schema might have changed - * @param dataSourcesToRebuild suspected dataSources for which the schema might have changed + * @param segmentsToRefresh segments for which the schema might have changed + * @param dataSourcesToRebuild dataSources for which the schema might have changed * @throws IOException when querying segment from data nodes and tasks */ @Override From d6ac350500b2e40e12524adf7623eb0969d9d360 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 16:13:57 +0530 Subject: [PATCH 40/82] Update base property name for query config classes in Coordinator --- .../main/java/org/apache/druid/cli/CliCoordinator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index d60bc67e4ce1..f355697024bf 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -471,10 +471,10 @@ public void configure(Binder binder) binder.install(new JoinableFactoryModule()); JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.coordinator.select", TierSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.coordinator.select.tier.custom", CustomTierSelectorStrategyConfig.class); - JsonConfigProvider.bind(binder, "druid.coordinator.balancer", ServerSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.coordinator.retryPolicy", RetryQueryRunnerConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.select", TierSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.select.tier.custom", CustomTierSelectorStrategyConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.balancer", ServerSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.retryPolicy", RetryQueryRunnerConfig.class); binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); binder.bind(CachingClusteredClient.class).in(LazySingleton.class); From 533236b1a0098717e67a7954bd4eeaa9b773909a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 15 Sep 2023 13:46:44 +0530 Subject: [PATCH 41/82] Log ds schema change when polling from coordinator --- .../AbstractSegmentMetadataCache.java | 30 ++++++------- .../CoordinatorSegmentMetadataCache.java | 8 ++-- .../metadata/DataSourceInformation.java | 2 +- .../schema/BrokerSegmentMetadataCache.java | 42 +++++++++++-------- 4 files changed, 44 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index 7cd3651533a6..f6cd12469ffe 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -90,7 +90,7 @@ /** * An abstract class that listens for segment change events and caches segment metadata and periodically refreshes - * the segments and dataSources. + * the segments and datasources. * *

    This class is generic and is parameterized by a type {@code T} that extends {@link DataSourceInformation}.

    * @@ -192,7 +192,7 @@ public abstract class AbstractSegmentMetadataCache tables = new ConcurrentHashMap<>(); @@ -207,7 +207,7 @@ public abstract class AbstractSegmentMetadataCache dataSourcesNeedingRebuild = new HashSet<>(); @@ -352,7 +352,7 @@ private void startCacheExec() log.warn(e, "Metadata refresh failed, trying again soon."); synchronized (lock) { - // Add our segments and dataSources back to their refresh and rebuild lists. + // Add our segments and datasources back to their refresh and rebuild lists. segmentsNeedingRefresh.addAll(segmentsToRefresh); dataSourcesNeedingRebuild.addAll(dataSourcesToRebuild); lastFailure = System.currentTimeMillis(); @@ -408,11 +408,11 @@ public void awaitInitialization() throws InterruptedException } /** - * Fetch schema for the given dataSource. + * Fetch schema for the given datasource. * - * @param name dataSource + * @param name datasource * - * @return schema information for the given dataSource + * @return schema information for the given datasource */ public T getDatasource(String name) { @@ -420,7 +420,7 @@ public T getDatasource(String name) } /** - * @return Map of dataSource and corresponding schema information. + * @return Map of datasource and corresponding schema information. */ public Map getDataSourceInformationMap() { @@ -428,7 +428,7 @@ public Map getDataSourceInformationMap() } /** - * @return Set of dataSources for which schema information is cached. + * @return Set of datasources for which schema information is cached. */ public Set getDatasourceNames() { @@ -452,7 +452,7 @@ public Map getSegmentMetadataSnapshot() /** * Get metadata for the specified segment, which includes information like RowSignature, realtime & numRows. * - * @param datasource segment dataSource + * @param datasource segment datasource * @param segmentId segment Id * * @return Metadata information for the given segment @@ -479,7 +479,7 @@ public int getTotalSegments() * The child classes must override this method with the logic to build and cache table schema. * * @param segmentsToRefresh segments for which the schema might have changed - * @param dataSourcesToRebuild dataSources for which the schema might have changed + * @param dataSourcesToRebuild datasources for which the schema might have changed * @throws IOException when querying segment schema from data nodes and tasks */ public abstract void refresh(Set segmentsToRefresh, Set dataSourcesToRebuild) throws IOException; @@ -695,7 +695,7 @@ public Set refreshSegments(final Set segments) throws IOEx { final Set retVal = new HashSet<>(); - // Organize segments by dataSource. + // Organize segments by datasource. final Map> segmentMap = new TreeMap<>(); for (SegmentId segmentId : segments) { @@ -742,7 +742,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin throw new ISE("'segments' must all match 'dataSource'!"); } - log.debug("Refreshing metadata for dataSource[%s].", dataSource); + log.debug("Refreshing metadata for datasource[%s].", dataSource); final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); @@ -821,7 +821,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin emitter.emit(builder.setMetric("metadatacache/refresh/time", refreshDurationMillis)); log.debug( - "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", + "Refreshed metadata for datasource [%s] in %,d ms (%d segments queried, %d segments left).", dataSource, refreshDurationMillis, retVal.size(), @@ -899,7 +899,7 @@ public Sequence runSegmentMetadataQuery( final Iterable segments ) { - // Sanity check: getOnlyElement of a set, to ensure all segments have the same dataSource. + // Sanity check: getOnlyElement of a set, to ensure all segments have the same datasource. final String dataSource = Iterables.getOnlyElement( StreamSupport.stream(segments.spliterator(), false) .map(SegmentId::getDataSource).collect(Collectors.toSet()) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index a268c25c4790..ff57ade02429 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -61,7 +61,7 @@ public CoordinatorSegmentMetadataCache( * The schema information for individual segments is combined to construct a table schema, which is then cached. * * @param segmentsToRefresh segments for which the schema might have changed - * @param dataSourcesToRebuild dataSources for which the schema might have changed + * @param dataSourcesToRebuild datasources for which the schema might have changed * @throws IOException when querying segment from data nodes and tasks */ @Override @@ -74,17 +74,17 @@ public void refresh(final Set segmentsToRefresh, final Set da // Add missing segments back to the refresh list. segmentsNeedingRefresh.addAll(Sets.difference(segmentsToRefresh, refreshed)); - // Compute the list of dataSources to rebuild tables for. + // Compute the list of datasources to rebuild tables for. dataSourcesToRebuild.addAll(dataSourcesNeedingRebuild); refreshed.forEach(segment -> dataSourcesToRebuild.add(segment.getDataSource())); dataSourcesNeedingRebuild.clear(); } - // Rebuild the dataSources. + // Rebuild the datasources. for (String dataSource : dataSourcesToRebuild) { final RowSignature rowSignature = buildDruidTable(dataSource); if (rowSignature == null) { - log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + log.info("datasource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index e007fdb9af18..9cfa991ff08a 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -27,7 +27,7 @@ import java.util.Objects; /** - * Encapsulates information about a dataSource, such as its schema. + * Encapsulates information about a datasource, such as its schema. */ public class DataSourceInformation { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 63f5b8a91b72..dbb228c758fd 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -90,13 +90,13 @@ public BrokerSegmentMetadataCache( /** * Refreshes the set of segments in two steps: *
      - *
    • Polls the coordinator for the dataSource schema.
      • + *
    • Polls the coordinator for the datasource schema.
      • *
    • Refreshes the remaining set of segments by executing a SegmentMetadataQuery and - * builds dataSource schema by combining segment schema.
      • + * builds datasource schema by combining segment schema. *
    * * @param segmentsToRefresh segments for which the schema might have changed - * @param dataSourcesToRebuild dataSources for which the schema might have changed + * @param dataSourcesToRebuild datasources for which the schema might have changed * @throws IOException when querying segment schema from data nodes and tasks */ @Override @@ -108,7 +108,7 @@ public void refresh(final Set segmentsToRefresh, final Set da final Map polledDataSourceMetadata = new HashMap<>(); - // Fetch dataSource information from the Coordinator + // Fetch datasource information from the Coordinator try { FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) .forEach(dataSourceInformation -> polledDataSourceMetadata.put( @@ -120,15 +120,16 @@ public void refresh(final Set segmentsToRefresh, final Set da )); } catch (Exception e) { - log.warn("Failed to query dataSource information from the Coordinator."); + log.warn("Failed to query datasource information from the Coordinator."); } - // remove any extra dataSources returned + // remove any extra datasources returned polledDataSourceMetadata.keySet().removeIf(Predicates.not(dataSourcesToQuery::contains)); - tables.putAll(polledDataSourceMetadata); + // update datasource metadata in the cache + polledDataSourceMetadata.forEach(this::updateDsMetadata); - // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. + // Remove segments of the datasource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); // Refresh the remaining segments. @@ -138,31 +139,36 @@ public void refresh(final Set segmentsToRefresh, final Set da // Add missing segments back to the refresh list. segmentsNeedingRefresh.addAll(Sets.difference(segmentsToRefresh, refreshed)); - // Compute the list of dataSources to rebuild tables for. + // Compute the list of datasources to rebuild tables for. dataSourcesToRebuild.addAll(dataSourcesNeedingRebuild); refreshed.forEach(segment -> dataSourcesToRebuild.add(segment.getDataSource())); - // Remove those dataSource for which we received schema from the Coordinator. + // Remove those datasource for which we received schema from the Coordinator. dataSourcesToRebuild.removeAll(polledDataSourceMetadata.keySet()); dataSourcesNeedingRebuild.clear(); } - // Rebuild the dataSources. + // Rebuild the datasources. for (String dataSource : dataSourcesToRebuild) { final RowSignature rowSignature = buildDruidTable(dataSource); if (rowSignature == null) { - log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + log.info("datasource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } final PhysicalDatasourceMetadata physicalDatasourceMetadata = dataSourceMetadataFactory.build(dataSource, rowSignature); - final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); - if (oldTable == null || !oldTable.getRowSignature().equals(physicalDatasourceMetadata.getRowSignature())) { - log.info("[%s] has new signature: %s.", dataSource, rowSignature); - } else { - log.debug("[%s] signature is unchanged.", dataSource); - } + updateDsMetadata(dataSource, physicalDatasourceMetadata); + } + } + + private void updateDsMetadata(String dataSource, PhysicalDatasourceMetadata physicalDatasourceMetadata) + { + final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); + if (oldTable == null || !oldTable.getRowSignature().equals(physicalDatasourceMetadata.getRowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, physicalDatasourceMetadata.getRowSignature()); + } else { + log.debug("[%s] signature is unchanged.", dataSource); } } } From 70f0888f470f62c50d60d72a9a8e99b3acb28552 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 15 Sep 2023 13:49:34 +0530 Subject: [PATCH 42/82] update the logic to determine is_active status in segments table for segment polled from coordiantor --- .../org/apache/druid/sql/calcite/schema/SystemSchema.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 26382b9e1c68..9bbd964239c3 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -317,8 +317,8 @@ public Enumerable scan(DataContext root) // so realtime segments are not published and vice versa boolean isPublished = !val.isRealtime(); - // is_active is true for published segments that are not overshadowed - boolean isActive = isPublished && !val.isOvershadowed(); + // is_active is true for published segments that are not overshadowed or else they should be realtime + boolean isActive = isPublished ? !val.isOvershadowed() : val.isRealtime(); try { return new Object[]{ From b32dfd69ee0ba06854b09209a1c9c0ad87ac2c5b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 15 Sep 2023 15:27:11 +0530 Subject: [PATCH 43/82] Update the logic to set numRows in the sys segments table, add comments --- .../druid/sql/calcite/schema/SystemSchema.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 9bbd964239c3..fa0b6babe12b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -292,8 +292,7 @@ public Enumerable scan(DataContext root) } // Get segments from metadata segment cache (if enabled in SQL planner config), else directly from - // Coordinator. - // this may include both published and realtime segments. + // Coordinator. This may include both published and realtime segments. final Iterator metadataStoreSegments = metadataView.getSegments(); final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(druidSchema.cache().getTotalSegments()); @@ -305,15 +304,23 @@ public Enumerable scan(DataContext root) segmentsAlreadySeen.add(segment.getId()); final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(segment.getId()); long numReplicas = 0L, numRows = 0L, isRealtime, isAvailable = 0L; + if (partialSegmentData != null) { numReplicas = partialSegmentData.getNumReplicas(); - numRows = partialSegmentData.getNumRows(); isAvailable = partialSegmentData.isAvailable(); + numRows = partialSegmentData.getNumRows(); + } + + // If table schema building is enabled on the Coordinator, SegmentMetadataCache on the + // broker might have outdated or no information regarding numRows and rowSignature for a segment. + // We should use {@code numRows} from the segment polled from the coordinator. + if (null != val.getNumRows()) { + numRows = val.getNumRows(); } isRealtime = Boolean.TRUE.equals(val.isRealtime()) ? 1 : 0; - // set of segments returned from coordinator include published and realtime segments + // set of segments returned from Coordinator include published and realtime segments // so realtime segments are not published and vice versa boolean isPublished = !val.isRealtime(); @@ -351,6 +358,8 @@ public Enumerable scan(DataContext root) } }); + // When table schema building is enabled on the Coordinator, all the segments in this loop + // would be covered in the previous iteration since Coordinator would return realtime segments as well. final FluentIterable availableSegments = FluentIterable .from(() -> getAuthorizedAvailableSegments( availableSegmentEntries, @@ -399,7 +408,6 @@ public Enumerable scan(DataContext root) ); return Linq4j.asEnumerable(allSegments).where(Objects::nonNull); - } private Iterator getAuthorizedPublishedSegments( From 17417b512e4093108ad2a9e9c7aacc9806abe562 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 15 Sep 2023 15:35:59 +0530 Subject: [PATCH 44/82] Rename config druid.coordinator.segmentMetadataCache.enabled to druid.coordinator.centralizedSchemaManagement.enabled --- .../src/main/java/org/apache/druid/cli/CliCoordinator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index f355697024bf..18e612953362 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -148,7 +148,7 @@ public class CliCoordinator extends ServerRunnable { private static final Logger log = new Logger(CliCoordinator.class); private static final String AS_OVERLORD_PROPERTY = "druid.coordinator.asOverlord.enabled"; - private static final String SEGMENT_METADATA_CACHE_ENABLED = "druid.coordinator.segmentMetadataCache.enabled"; + private static final String CENTRALIZED_SCHEMA_MANAGEMENT_ENABLED = "druid.coordinator.centralizedSchemaManagement.enabled"; private Properties properties; private boolean beOverlord; @@ -360,7 +360,7 @@ public static boolean isOverlord(Properties properties) private boolean isSegmentMetadataCacheEnabled() { - return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED)); + return Boolean.parseBoolean(properties.getProperty(CENTRALIZED_SCHEMA_MANAGEMENT_ENABLED)); } private static class CoordinatorCustomDutyGroupsProvider implements Provider From 907ace3c379fe16b0428f8f9567c02a691fc85b0 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 20 Sep 2023 12:42:25 +0530 Subject: [PATCH 45/82] Report cache init time irrespective of the awaitInitializationOnStart config --- .../AbstractSegmentMetadataCache.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index f6cd12469ffe..883cf420b72b 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -285,6 +285,7 @@ private void startCacheExec() { cacheExec.submit( () -> { + final Stopwatch stopwatch = Stopwatch.createStarted(); long lastRefresh = 0L; long lastFailure = 0L; @@ -322,7 +323,7 @@ private void startCacheExec() if (isServerViewInitialized && lastFailure == 0L) { // Server view is initialized, but we don't need to do a refresh. Could happen if there are // no segments in the system yet. Just mark us as initialized, then. - initialized.countDown(); + setInitialized(stopwatch); } // Wait some more, we'll wake up when it might be time to do another refresh. @@ -342,7 +343,7 @@ private void startCacheExec() refresh(segmentsToRefresh, dataSourcesToRebuild); - initialized.countDown(); + setInitialized(stopwatch); } catch (InterruptedException e) { // Fall through. @@ -376,22 +377,26 @@ private void startCacheExec() ); } + private void setInitialized(Stopwatch stopwatch) + { + // report the cache init time + if (initialized.getCount() == 1) { + long elapsedTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); + emitter.emit(ServiceMetricEvent.builder().setMetric("metadatacache/init/time", elapsedTime)); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), elapsedTime); + stopwatch.stop(); + } + initialized.countDown(); + } + @LifecycleStart public void start() throws InterruptedException { - log.info("Starting SegmentMetadataCache."); + log.info("%s waiting for initialization.", getClass().getSimpleName()); startCacheExec(); if (config.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); awaitInitialization(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().setMetric( - "metadatacache/init/time", - endMillis - startMillis - )); } } From 441f37a8cbe3c1affbf17bf1884a93f25db835e5 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 20 Sep 2023 18:46:22 +0530 Subject: [PATCH 46/82] Report metric for fetching schema from coordinator --- .../AbstractSegmentMetadataCache.java | 3 +- .../schema/BrokerSegmentMetadataCache.java | 46 +++++++++++++------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index 883cf420b72b..87ad82ef4151 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -118,7 +118,6 @@ public abstract class AbstractSegmentMetadataCache segmentsToRefresh, final Set da segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); - final Map polledDataSourceMetadata = new HashMap<>(); - // Fetch datasource information from the Coordinator - try { - FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) - .forEach(dataSourceInformation -> polledDataSourceMetadata.put( - dataSourceInformation.getDataSource(), - dataSourceMetadataFactory.build( - dataSourceInformation.getDataSource(), - dataSourceInformation.getRowSignature() - ) - )); - } - catch (Exception e) { - log.warn("Failed to query datasource information from the Coordinator."); - } + Map polledDataSourceMetadata = queryDataSourceInformation(dataSourcesToQuery); // remove any extra datasources returned polledDataSourceMetadata.keySet().removeIf(Predicates.not(dataSourcesToQuery::contains)); @@ -162,6 +151,33 @@ public void refresh(final Set segmentsToRefresh, final Set da } } + private Map queryDataSourceInformation(Set dataSourcesToQuery) + { + final Map polledDataSourceMetadata = new HashMap<>(); + + Stopwatch stopwatch = Stopwatch.createStarted(); + + try { + FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) + .forEach(dataSourceInformation -> polledDataSourceMetadata.put( + dataSourceInformation.getDataSource(), + dataSourceMetadataFactory.build( + dataSourceInformation.getDataSource(), + dataSourceInformation.getRowSignature() + ) + )); + } + catch (Exception e) { + log.warn("Failed to query datasource information from the Coordinator."); + } + + emitter.emit(ServiceMetricEvent.builder().setMetric( + "metadatacache/schemaPoll/time", + stopwatch.elapsed(TimeUnit.MILLISECONDS))); + + return polledDataSourceMetadata; + } + private void updateDsMetadata(String dataSource, PhysicalDatasourceMetadata physicalDatasourceMetadata) { final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); From bd5b0483f4ed776b57996599b782cf9aa3fa71a8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 21 Sep 2023 15:45:24 +0530 Subject: [PATCH 47/82] Add auth check in api to return dataSourceInformation, report metrics for broker-coordinator communication --- .../apache/druid/server/http/MetadataResource.java | 12 +++++++++++- .../druid/server/http/MetadataResourceTest.java | 2 +- .../calcite/schema/BrokerSegmentMetadataCache.java | 4 ++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index b6f99e614337..f7bf3710ffef 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -371,6 +371,7 @@ public Response getSegment( @Path("/dataSourceInformation") @Produces(MediaType.APPLICATION_JSON) public Response getDataSourceInformation( + HttpServletRequest req, List dataSources ) { @@ -389,6 +390,15 @@ public Response getDataSourceInformation( } } - return Response.status(Response.Status.OK).entity(results).build(); + final Function> raGenerator = dataSourceInformation -> Collections + .singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(dataSourceInformation.getDataSource())); + + final Iterable authorizedDataSourceInformation = AuthorizationUtils.filterAuthorizedResources( + req, + results, + raGenerator, + authorizerMapper + ); + return Response.status(Response.Status.OK).entity(authorizedDataSourceInformation).build(); } } diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index a430081c10a7..cb6e9d2a37ed 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -274,7 +274,7 @@ public void testGetDataSourceInformation() coordinatorSegmentMetadataCache ); - Response response = metadataResource.getDataSourceInformation(Collections.singletonList(DATASOURCE1)); + Response response = metadataResource.getDataSourceInformation(request, Collections.singletonList(DATASOURCE1)); List dataSourceInformations = extractResponseList(response); Assert.assertEquals(dataSourceInformations.size(), 1); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index c0f63846e1c3..08f89212a670 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -158,6 +158,8 @@ private Map queryDataSourceInformation(Set polledDataSourceMetadata.put( dataSourceInformation.getDataSource(), @@ -169,6 +171,8 @@ private Map queryDataSourceInformation(Set Date: Thu, 21 Sep 2023 15:57:59 +0530 Subject: [PATCH 48/82] Fix bug in Coordinator api to return dataSourceInformation --- .../java/org/apache/druid/server/http/MetadataResource.java | 2 +- .../druid/sql/calcite/schema/BrokerSegmentMetadataCache.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index f7bf3710ffef..9676f4366d16 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -371,7 +371,7 @@ public Response getSegment( @Path("/dataSourceInformation") @Produces(MediaType.APPLICATION_JSON) public Response getDataSourceInformation( - HttpServletRequest req, + @Context final HttpServletRequest req, List dataSources ) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 08f89212a670..e3e011309214 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -170,7 +170,7 @@ private Map queryDataSourceInformation(Set Date: Thu, 21 Sep 2023 16:04:58 +0530 Subject: [PATCH 49/82] Minor change --- .../java/org/apache/druid/server/http/MetadataResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 9676f4366d16..d688a106130c 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -372,7 +372,7 @@ public Response getSegment( @Produces(MediaType.APPLICATION_JSON) public Response getDataSourceInformation( @Context final HttpServletRequest req, - List dataSources + final List dataSources ) { // if {@code coordinatorSegmentMetadataCache} is null, implies the feature is disabled. Return NOT_FOUND. From 5d161488d0f68dafebb9dbc35af2530adbffa2c1 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 23 Sep 2023 13:30:48 +0530 Subject: [PATCH 50/82] Address comments around docs, minor renaming --- .../apache/druid/client/BrokerServerView.java | 2 ++ .../druid/client/CoordinatorServerView.java | 2 +- .../druid/client/CoordinatorTimeline.java | 7 +++--- .../QueryableCoordinatorServerView.java | 22 +++++++++---------- .../AbstractSegmentMetadataCache.java | 11 +++++----- .../server/http/DataSourcesResource.java | 2 +- .../druid/server/http/MetadataResource.java | 2 +- .../server/http/DataSourcesResourceTest.java | 8 +++---- .../schema/BrokerSegmentMetadataCache.java | 12 +++++----- .../sql/calcite/util/QueryFrameworkUtils.java | 1 - 10 files changed, 36 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index 77fc59219096..087346d7b6cc 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -85,8 +85,10 @@ public class BrokerServerView implements TimelineServerView protected final Object lock = new Object(); + // Map of segmentIds and the set of server where the segment is present protected final Map selectors = new HashMap<>(); + // Map of datasource and segment timeline protected final Map> timelines = new HashMap<>(); @Inject diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index bcc609ec9bff..114732949280 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -212,7 +212,7 @@ public VersionedIntervalTimeline getTimeline(DataSource } @Override - public Map getSegmentLoadInfos() + public Map getLoadInfoForAllSegments() { return segmentLoadInfos; } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java b/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java index 621d981551b8..b5041ef8e094 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java @@ -26,17 +26,18 @@ import java.util.Map; /** - * Segment timeline maintained in the coordinator. + * Segment timeline maintained in the coordinator. It provides methods for retrieving timeline information + * related to datasources and {@link SegmentLoadInfo}. */ public interface CoordinatorTimeline extends InventoryView { /** - * Retrieve timeline for a dataSource. + * Retrieve timeline for a datasource. */ VersionedIntervalTimeline getTimeline(DataSource dataSource); /** * Server information for all segments in the timeline. */ - Map getSegmentLoadInfos(); + Map getLoadInfoForAllSegments(); } diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 68073ade2027..164ce9dceb5f 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -62,10 +62,10 @@ public QueryableCoordinatorServerView( final QueryWatcher queryWatcher, final @Smile ObjectMapper smileMapper, final @EscalatedClient HttpClient httpClient, - FilteredServerInventoryView baseView, - TierSelectorStrategy tierSelectorStrategy, - ServiceEmitter emitter, - CoordinatorSegmentWatcherConfig segmentWatcherConfig + final FilteredServerInventoryView baseView, + final TierSelectorStrategy tierSelectorStrategy, + final ServiceEmitter emitter, + final CoordinatorSegmentWatcherConfig segmentWatcherConfig ) { super(warehouse, queryWatcher, smileMapper, httpClient, baseView, tierSelectorStrategy, emitter, new BrokerSegmentWatcherConfig() { @@ -89,27 +89,27 @@ public boolean isAwaitInitializationOnStart() public VersionedIntervalTimeline getTimeline(DataSource dataSource) { String table = Iterables.getOnlyElement(dataSource.getTableNames()); - VersionedIntervalTimeline timeline; + VersionedIntervalTimeline baseTimeline; synchronized (lock) { - timeline = timelines.get(table); + baseTimeline = timelines.get(table); } - VersionedIntervalTimeline newTimeline = + VersionedIntervalTimeline segmentLoadInfoTimeline = new VersionedIntervalTimeline<>(Comparator.naturalOrder()); - newTimeline.addAll( - timeline.iterateAllObjects().stream() + segmentLoadInfoTimeline.addAll( + baseTimeline.iterateAllObjects().stream() .map(serverSelector -> new VersionedIntervalTimeline.PartitionChunkEntry<>( serverSelector.getSegment().getInterval(), serverSelector.getSegment().getVersion(), serverSelector.getSegment().getShardSpec().createChunk(serverSelector.toSegmentLoadInfo()) )).iterator()); - return newTimeline; + return segmentLoadInfoTimeline; } @Override - public Map getSegmentLoadInfos() + public Map getLoadInfoForAllSegments() { return CollectionUtils.mapValues(selectors, ServerSelector::toSegmentLoadInfo); } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index 87ad82ef4151..1095e5473c68 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -89,8 +89,9 @@ import java.util.stream.StreamSupport; /** - * An abstract class that listens for segment change events and caches segment metadata and periodically refreshes - * the segments and datasources. + * An abstract class that listens for segment change events and caches segment metadata. It periodically refreshes + * the segments, by fetching their metadata which includes schema information from sources like + * data nodes, db (the logic is specificed in the child class) and builds table schema. * *

    This class is generic and is parameterized by a type {@code T} that extends {@link DataSourceInformation}.

    * @@ -324,7 +325,7 @@ private void startCacheExec() if (isServerViewInitialized && lastFailure == 0L) { // Server view is initialized, but we don't need to do a refresh. Could happen if there are // no segments in the system yet. Just mark us as initialized, then. - setInitialized(stopwatch); + setInitializedAndReportInitTime(stopwatch); } // Wait some more, we'll wake up when it might be time to do another refresh. @@ -344,7 +345,7 @@ private void startCacheExec() refresh(segmentsToRefresh, dataSourcesToRebuild); - setInitialized(stopwatch); + setInitializedAndReportInitTime(stopwatch); } catch (InterruptedException e) { // Fall through. @@ -378,7 +379,7 @@ private void startCacheExec() ); } - private void setInitialized(Stopwatch stopwatch) + private void setInitializedAndReportInitTime(Stopwatch stopwatch) { // report the cache init time if (initialized.getCount() == 1) { diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index c8e998e72cfa..dfbd559c1b06 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -487,7 +487,7 @@ public Response getDatasourceLoadstatus( private SegmentsLoadStatistics computeSegmentLoadStatistics(Iterable segments) { - Map segmentLoadInfos = serverInventoryView.getSegmentLoadInfos(); + Map segmentLoadInfos = serverInventoryView.getLoadInfoForAllSegments(); int numPublishedSegments = 0; int numUnavailableSegments = 0; int numLoadedSegments = 0; diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index d688a106130c..7180cb0cfb2a 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -365,7 +365,7 @@ public Response getSegment( * API to fetch {@link DataSourceInformation} for the specified dataSources. * * @param dataSources list of dataSources to be queried - * @return information including schema details for the specified dataSources + * @return information including schema details for the specified datasources */ @POST @Path("/dataSourceInformation") diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 551151458546..c33fffc80830 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -1302,7 +1302,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1318,7 +1318,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1381,7 +1381,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1397,7 +1397,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index e3e011309214..ecae501f591d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -46,14 +46,14 @@ import java.util.concurrent.TimeUnit; /** - * Broker-side cache of segment metadata that combines segments to identify - * dataSources which become "tables" in Calcite. This cache provides the "physical" + * Broker-side cache of segment metadata that combines segments to build + * datasources which become "tables" in Calcite. This cache provides the "physical" * metadata about a dataSource which is blended with catalog "logical" metadata * to provide the final user-view of each dataSource. *

    * This class extends {@link AbstractSegmentMetadataCache} and introduces following changes, *

      - *
    • The refresh mechanism now includes polling the coordinator for dataSource schema, + *
    • The refresh mechanism includes polling the coordinator for datasource schema, * and falling back to running {@link org.apache.druid.query.metadata.metadata.SegmentMetadataQuery}.
      • *
    • It builds and caches {@link PhysicalDatasourceMetadata} object for the table schema
      • *
    @@ -116,7 +116,7 @@ public void refresh(final Set segmentsToRefresh, final Set da polledDataSourceMetadata.keySet().removeIf(Predicates.not(dataSourcesToQuery::contains)); // update datasource metadata in the cache - polledDataSourceMetadata.forEach(this::updateDsMetadata); + polledDataSourceMetadata.forEach(this::updateDSMetadata); // Remove segments of the datasource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); @@ -147,7 +147,7 @@ public void refresh(final Set segmentsToRefresh, final Set da } final PhysicalDatasourceMetadata physicalDatasourceMetadata = dataSourceMetadataFactory.build(dataSource, rowSignature); - updateDsMetadata(dataSource, physicalDatasourceMetadata); + updateDSMetadata(dataSource, physicalDatasourceMetadata); } } @@ -182,7 +182,7 @@ private Map queryDataSourceInformation(Set Date: Sat, 23 Sep 2023 14:00:51 +0530 Subject: [PATCH 51/82] Remove null check from MetadataResource#getDataSourceInformation --- .../java/org/apache/druid/server/http/MetadataResource.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 7180cb0cfb2a..c300cd13504a 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -382,10 +382,9 @@ public Response getDataSourceInformation( Map dataSourceSchemaMap = coordinatorSegmentMetadataCache.getDataSourceInformationMap(); List results = new ArrayList<>(); - List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; for (Map.Entry entry : dataSourceSchemaMap.entrySet()) { - if (dataSourcesToRetain.contains(entry.getKey())) { + if (dataSources.contains(entry.getKey())) { results.add(entry.getValue()); } } From e129d3e63ec7bcd398d76aa967df7e8f87fd8246 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 25 Sep 2023 14:32:18 +0530 Subject: [PATCH 52/82] Install cache module in Coordinator, if feature is enabled and beOverlord=false --- .../java/org/apache/druid/cli/CliCoordinator.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 18e612953362..353b981c506e 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -50,6 +50,7 @@ import org.apache.druid.client.selector.TierSelectorStrategy; import org.apache.druid.discovery.NodeRole; import org.apache.druid.guice.BrokerProcessingModule; +import org.apache.druid.guice.CacheModule; import org.apache.druid.guice.ConfigProvider; import org.apache.druid.guice.Jerseys; import org.apache.druid.guice.JoinableFactoryModule; @@ -220,7 +221,7 @@ public void configure(Binder binder) } if (isSegmentMetadataCacheEnabled()) { - binder.install(new CoordinatorSegmentMetadataCacheModule()); + binder.install(new CoordinatorSegmentMetadataCacheModule(beOverlord)); } else { binder.bind(CoordinatorTimeline.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); @@ -458,6 +459,13 @@ public Supplier> get() private static class CoordinatorSegmentMetadataCacheModule implements Module { + private final boolean beOverlord; + + public CoordinatorSegmentMetadataCacheModule(boolean beOverlord) + { + this.beOverlord = beOverlord; + } + @Override public void configure(Binder binder) { @@ -469,6 +477,9 @@ public void configure(Binder binder) binder.install(new QueryableModule()); binder.install(new BrokerProcessingModule()); binder.install(new JoinableFactoryModule()); + if (!beOverlord) { + binder.install(new CacheModule()); + } JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.query.select", TierSelectorStrategy.class); From 01c27c9876bb5adaeea0f0463fc6fa47c0eb5599 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 29 Sep 2023 11:28:52 +0530 Subject: [PATCH 53/82] Minor change in QueryableCoordinatorServerView --- .../main/java/org/apache/druid/client/BrokerServerView.java | 2 +- .../apache/druid/client/QueryableCoordinatorServerView.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index 087346d7b6cc..a943d1025757 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -76,12 +76,12 @@ public class BrokerServerView implements TimelineServerView private final QueryWatcher queryWatcher; private final ObjectMapper smileMapper; private final HttpClient httpClient; - private final FilteredServerInventoryView baseView; private final TierSelectorStrategy tierSelectorStrategy; private final ServiceEmitter emitter; private final BrokerSegmentWatcherConfig segmentWatcherConfig; private final Predicate> segmentFilter; private final CountDownLatch initialized = new CountDownLatch(1); + protected final FilteredServerInventoryView baseView; protected final Object lock = new Object(); diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 164ce9dceb5f..a35f218d19ac 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -54,8 +54,6 @@ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline { - private final FilteredServerInventoryView baseView; - @Inject public QueryableCoordinatorServerView( final QueryToolChestWarehouse warehouse, @@ -75,7 +73,6 @@ public boolean isAwaitInitializationOnStart() return segmentWatcherConfig.isAwaitInitializationOnStart(); } }); - this.baseView = baseView; } /** From 87c9873be2925961123edb6f4025a23b037c2c2a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 14 Oct 2023 18:01:25 +0530 Subject: [PATCH 54/82] Remove QueryableCoordinatorServerView, add a new QuerySegmentWalker implementation for segment metadata queries on coordinator --- docs/api-reference/legacy-metadata-api.md | 12 + docs/operations/metrics.md | 6 + .../DatasourceOptimizerTest.java | 10 +- .../apache/druid/client/BrokerServerView.java | 36 +- .../druid/client/CachingClusteredClient.java | 19 +- .../druid/client/CoordinatorServerView.java | 85 ++- .../druid/client/CoordinatorTimeline.java | 43 -- .../client/DirectDruidClientFactory.java | 70 +++ .../QueryableCoordinatorServerView.java | 137 ----- .../apache/druid/client/SegmentLoadInfo.java | 11 + .../druid/client/selector/ServerSelector.java | 19 - .../AbstractSegmentMetadataCache.java | 64 +-- .../CoordinatorSegmentMetadataCache.java | 58 +- .../SegmentMetadataQuerySegmentWalker.java | 259 +++++++++ .../server/http/DataSourcesResource.java | 8 +- .../druid/client/BrokerServerViewTest.java | 9 +- .../client/CachingClusteredClientTest.java | 10 +- ...st.java => CoordinatorServerViewTest.java} | 242 +++++---- ...natorSegmentDataCacheConcurrencyTest.java} | 109 ++-- ...CoordinatorSegmentMetadataCacheCommon.java | 78 +++ .../CoordinatorSegmentMetadataCacheTest.java | 161 +++++- .../metadata/SegmentMetadataCacheCommon.java | 244 ++------- ...SegmentMetadataQuerySegmentWalkerTest.java | 343 ++++++++++++ .../metadata/TestCoordinatorServerView.java | 237 +++++++++ .../TestSegmentMetadataQueryWalker.java | 121 +++++ .../CuratorDruidCoordinatorTest.java | 2 +- .../org/apache/druid/cli/CliCoordinator.java | 105 ++-- .../schema/BrokerSegmentMetadataCache.java | 52 +- .../BrokerSegmentMetadataCacheCommon.java | 100 ++++ ...erSegmentMetadataCacheConcurrencyTest.java | 502 ++++++++++++++++++ .../BrokerSegmentMetadataCacheTest.java | 185 +++++-- .../schema/DruidSchemaNoDataInitTest.java | 2 +- .../sql/calcite/schema/SystemSchemaTest.java | 2 +- .../druid/sql/calcite/util/CalciteTests.java | 1 - .../sql/calcite/util/QueryFrameworkUtils.java | 1 - .../calcite/util}/TestTimelineServerView.java | 2 +- 36 files changed, 2557 insertions(+), 788 deletions(-) delete mode 100644 server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java create mode 100644 server/src/main/java/org/apache/druid/client/DirectDruidClientFactory.java delete mode 100644 server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java create mode 100644 server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalker.java rename server/src/test/java/org/apache/druid/client/{CoordinatorTimelineTest.java => CoordinatorServerViewTest.java} (69%) rename server/src/test/java/org/apache/druid/segment/metadata/{SegmentDataCacheConcurrencyTest.java => CoordinatorSegmentDataCacheConcurrencyTest.java} (86%) create mode 100644 server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheCommon.java create mode 100644 server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java create mode 100644 server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java create mode 100644 server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheCommon.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java rename {server/src/test/java/org/apache/druid/segment/metadata => sql/src/test/java/org/apache/druid/sql/calcite/util}/TestTimelineServerView.java (99%) diff --git a/docs/api-reference/legacy-metadata-api.md b/docs/api-reference/legacy-metadata-api.md index fe031c7348aa..bef4152e485c 100644 --- a/docs/api-reference/legacy-metadata-api.md +++ b/docs/api-reference/legacy-metadata-api.md @@ -116,10 +116,18 @@ Returns a list of all segments for one or more specific datasources enabled in t Returns a list of all segments for each datasource with the full segment metadata and an extra field `overshadowed`. +`GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments` + +Additionally, returns the realtime segments for all datasources, with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. + `GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&datasources={dataSourceName1}&datasources={dataSourceName2}` Returns a list of all segments for one or more specific datasources with the full segment metadata and an extra field `overshadowed`. +`GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments&datasources={dataSourceName1}&datasources={dataSourceName2}` + +Additionally, returns the realtime segments for the speicified datasources, with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. + `GET /druid/coordinator/v1/metadata/datasources` Returns a list of the names of datasources with at least one used segment in the cluster, retrieved from the metadata database. Users should call this API to get the eventual state that the system will be in. @@ -166,6 +174,10 @@ Returns a list of all segments, overlapping with any of given intervals, for a Returns a list of all segments, overlapping with any of given intervals, for a datasource with the full segment metadata as stored in the metadata store. Request body is array of string ISO 8601 intervals like `[interval1, interval2,...]`—for example, `["2012-01-01T00:00:00.000/2012-01-03T00:00:00.000", "2012-01-05T00:00:00.000/2012-01-07T00:00:00.000"]`. +`POST /druid/coordinator/v1/metadata/dataSourceInformation` + +Returns information about the specified datasources, including the datasource schema. + ## Datasources diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md index c2a77a4f3d67..f7b5cb0edb89 100644 --- a/docs/operations/metrics.md +++ b/docs/operations/metrics.md @@ -71,6 +71,9 @@ Most metric values reset each emission period, as specified in `druid.monitoring |`metadatacache/init/time`|Time taken to initialize the broker segment metadata cache. Useful to detect if brokers are taking too long to start||Depends on the number of segments.| |`metadatacache/refresh/count`|Number of segments to refresh in broker segment metadata cache.|`dataSource`| |`metadatacache/refresh/time`|Time taken to refresh segments in broker segment metadata cache.|`dataSource`| +|`metadatacache/schemaPoll/count`|Number of coordinator polls to fetch datasource schema.|`dataSource`| +|`metadatacache/schemaPoll/failed`|Number of failed coordinator polls to fetch datasource schema.|`dataSource`| +|`metadatacache/schemaPoll/time`|Time taken for coordinator polls to fetch datasource schema.|`dataSource`| |`serverview/sync/healthy`|Sync status of the Broker with a segment-loading server such as a Historical or Peon. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled. This metric can be used in conjunction with `serverview/sync/unstableTime` to debug slow startup of Brokers.|`server`, `tier`|1 for fully synced servers, 0 otherwise| |`serverview/sync/unstableTime`|Time in milliseconds for which the Broker has been failing to sync with a segment-loading server. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled.|`server`, `tier`|Not emitted for synced servers.| |`subquery/rowLimit/count`|Number of subqueries whose results are materialized as rows (Java objects on heap).|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | @@ -354,6 +357,9 @@ These metrics are for the Druid Coordinator and are reset each time the Coordina |`serverview/init/time`|Time taken to initialize the coordinator server view.||Depends on the number of segments.| |`serverview/sync/healthy`|Sync status of the Coordinator with a segment-loading server such as a Historical or Peon. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled. You can use this metric in conjunction with `serverview/sync/unstableTime` to debug slow startup of the Coordinator.|`server`, `tier`|1 for fully synced servers, 0 otherwise| |`serverview/sync/unstableTime`|Time in milliseconds for which the Coordinator has been failing to sync with a segment-loading server. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled.|`server`, `tier`|Not emitted for synced servers.| +|`metadatacache/init/time`|Time taken to initialize the coordinator segment metadata cache.|`dataSource`|Depends on the number of segments.| +|`metadatacache/refresh/count`|Number of segments to refresh in coordinator segment metadata cache.|`dataSource`| +|`metadatacache/refresh/time`|Time taken to refresh segments in coordinator segment metadata cache.|`dataSource`| ## General Health diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java index 80ade5c1c9c7..1a6e462172b1 100644 --- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java +++ b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java @@ -31,6 +31,7 @@ import org.apache.druid.client.BatchServerInventoryView; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; +import org.apache.druid.client.DirectDruidClientFactory; import org.apache.druid.client.DruidServer; import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; import org.apache.druid.client.selector.RandomServerSelectorStrategy; @@ -295,11 +296,16 @@ public CallbackAction segmentViewInitialized() } }; - brokerServerView = new BrokerServerView( + DirectDruidClientFactory druidClientFactory = new DirectDruidClientFactory( + new NoopServiceEmitter(), EasyMock.createMock(QueryToolChestWarehouse.class), EasyMock.createMock(QueryWatcher.class), getSmileMapper(), - EasyMock.createMock(HttpClient.class), + EasyMock.createMock(HttpClient.class) + ); + + brokerServerView = new BrokerServerView( + druidClientFactory, baseView, new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), new NoopServiceEmitter(), diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index a943d1025757..4579698a4201 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -19,7 +19,6 @@ package org.apache.druid.client; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; import com.google.common.collect.Ordering; import com.google.inject.Inject; @@ -27,8 +26,6 @@ import org.apache.druid.client.selector.ServerSelector; import org.apache.druid.client.selector.TierSelectorStrategy; import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.guice.annotations.EscalatedClient; -import org.apache.druid.guice.annotations.Smile; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.concurrent.Execs; @@ -36,10 +33,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryToolChestWarehouse; -import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -72,10 +66,7 @@ public class BrokerServerView implements TimelineServerView private final ConcurrentMap clients = new ConcurrentHashMap<>(); private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); - private final QueryToolChestWarehouse warehouse; - private final QueryWatcher queryWatcher; - private final ObjectMapper smileMapper; - private final HttpClient httpClient; + private final DirectDruidClientFactory druidClientFactory; private final TierSelectorStrategy tierSelectorStrategy; private final ServiceEmitter emitter; private final BrokerSegmentWatcherConfig segmentWatcherConfig; @@ -93,20 +84,14 @@ public class BrokerServerView implements TimelineServerView @Inject public BrokerServerView( - final QueryToolChestWarehouse warehouse, - final QueryWatcher queryWatcher, - final @Smile ObjectMapper smileMapper, - final @EscalatedClient HttpClient httpClient, + final DirectDruidClientFactory directDruidClientFactory, final FilteredServerInventoryView baseView, final TierSelectorStrategy tierSelectorStrategy, final ServiceEmitter emitter, final BrokerSegmentWatcherConfig segmentWatcherConfig ) { - this.warehouse = warehouse; - this.queryWatcher = queryWatcher; - this.smileMapper = smileMapper; - this.httpClient = httpClient; + this.druidClientFactory = directDruidClientFactory; this.baseView = baseView; this.tierSelectorStrategy = tierSelectorStrategy; this.emitter = emitter; @@ -235,7 +220,7 @@ private void validateSegmentWatcherConfig(BrokerSegmentWatcherConfig watcherConf private QueryableDruidServer addServer(DruidServer server) { - QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); + QueryableDruidServer retVal = new QueryableDruidServer<>(server, druidClientFactory.makeDirectClient(server)); QueryableDruidServer exists = clients.put(server.getName(), retVal); if (exists != null) { log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); @@ -244,19 +229,6 @@ private QueryableDruidServer addServer(DruidServer server) return retVal; } - private DirectDruidClient makeDirectClient(DruidServer server) - { - return new DirectDruidClient( - warehouse, - queryWatcher, - smileMapper, - httpClient, - server.getScheme(), - server.getHost(), - emitter - ); - } - private QueryableDruidServer removeServer(DruidServer server) { for (DataSegment segment : server.iterateAllSegments()) { diff --git a/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java b/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java index 19df276344ef..3f1679f756e7 100644 --- a/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java @@ -79,6 +79,7 @@ import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.Overshadowable; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.TimelineLookup; import org.apache.druid.timeline.TimelineObjectHolder; @@ -844,23 +845,23 @@ private byte[] computeQueryCacheKeyWithJoin() } } - private static class TimelineConverter implements UnaryOperator> + public static class TimelineConverter> implements UnaryOperator> { private final Iterable specs; - TimelineConverter(final Iterable specs) + public TimelineConverter(final Iterable specs) { this.specs = specs; } @Override - public TimelineLookup apply(TimelineLookup timeline) + public TimelineLookup apply(TimelineLookup timeline) { - final VersionedIntervalTimeline timeline2 = + final VersionedIntervalTimeline timeline2 = new VersionedIntervalTimeline<>(Ordering.natural(), true); - Iterator> unfilteredIterator = + Iterator> unfilteredIterator = Iterators.transform(specs.iterator(), spec -> toChunkEntry(timeline, spec)); - Iterator> iterator = Iterators.filter( + Iterator> iterator = Iterators.filter( unfilteredIterator, Objects::nonNull ); @@ -871,12 +872,12 @@ public TimelineLookup apply(TimelineLookup toChunkEntry( - TimelineLookup timeline, + private PartitionChunkEntry toChunkEntry( + TimelineLookup timeline, SegmentDescriptor spec ) { - PartitionChunk chunk = timeline.findChunk( + PartitionChunk chunk = timeline.findChunk( spec.getInterval(), spec.getVersion(), spec.getPartitionNumber() diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index 114732949280..f303211b3558 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -22,6 +22,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; import com.google.inject.Inject; +import org.apache.druid.client.TimelineServerView.TimelineCallback; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.lifecycle.LifecycleStart; @@ -29,49 +30,60 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DataSource; +import org.apache.druid.query.QueryRunner; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.VersionedIntervalTimeline; import org.apache.druid.timeline.partition.PartitionChunk; +import javax.annotation.Nullable; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; +import java.util.function.Function; /** * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class CoordinatorServerView implements CoordinatorTimeline +public class CoordinatorServerView implements InventoryView { private static final Logger log = new Logger(CoordinatorServerView.class); private final Object lock = new Object(); - private final Map segmentLoadInfos; private final Map> timelines; - + private final ConcurrentMap serverQueryRunners; + private final ConcurrentMap timelineCallbacks; private final ServerInventoryView baseView; private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; - private final CountDownLatch initialized = new CountDownLatch(1); private final ServiceEmitter emitter; + @Nullable + private final DirectDruidClientFactory druidClientFactory; @Inject public CoordinatorServerView( - ServerInventoryView baseView, - CoordinatorSegmentWatcherConfig segmentWatcherConfig, - ServiceEmitter emitter + final ServerInventoryView baseView, + final CoordinatorSegmentWatcherConfig segmentWatcherConfig, + final ServiceEmitter emitter, + @Nullable final DirectDruidClientFactory druidClientFactory ) { this.baseView = baseView; this.segmentWatcherConfig = segmentWatcherConfig; this.emitter = emitter; + this.druidClientFactory = druidClientFactory; this.segmentLoadInfos = new HashMap<>(); this.timelines = new HashMap<>(); + this.serverQueryRunners = new ConcurrentHashMap<>(); + this.timelineCallbacks = new ConcurrentHashMap<>(); ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); baseView.registerSegmentCallback( @@ -96,6 +108,7 @@ public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server public ServerView.CallbackAction segmentViewInitialized() { initialized.countDown(); + runTimelineCallbacks(TimelineCallback::timelineInitialized); return ServerView.CallbackAction.CONTINUE; } } @@ -136,6 +149,8 @@ private void removeServer(DruidServer server) for (DataSegment segment : server.iterateAllSegments()) { serverRemovedSegment(server.getMetadata(), segment); } + // remove QueryRunner for the server + serverQueryRunners.remove(server.getName()); } private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) @@ -162,7 +177,28 @@ private void serverAddedSegment(final DruidServerMetadata server, final DataSegm ); segmentLoadInfos.put(segmentId, segmentLoadInfo); } + + if (null != druidClientFactory) { + QueryRunner queryRunner = serverQueryRunners.get(server.getName()); + if (null == queryRunner) { + DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); + if (inventoryValue == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping addition of segment[%s].", + server.getName(), + segmentId + ); + return; + } else { + serverQueryRunners.put(server.getName(), druidClientFactory.makeDirectClient(inventoryValue)); + } + } + } + segmentLoadInfo.addServer(server); + + // segment added notification + runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); } } @@ -178,7 +214,12 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen log.warn("Told to remove non-existant segment[%s]", segmentId); return; } - segmentLoadInfo.removeServer(server); + + if (segmentLoadInfo.removeServer(server)) { + // server segment removed notification + runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } + if (segmentLoadInfo.isEmpty()) { VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); segmentLoadInfos.remove(segmentId); @@ -197,12 +238,37 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen segment.getInterval(), segment.getVersion() ); + } else { + // segment removed notification + runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); } } } } - @Override + public void registerTimelineCallback(final Executor exec, final TimelineCallback callback) + { + timelineCallbacks.put(callback, exec); + } + + private void runTimelineCallbacks(final Function function) + { + for (Map.Entry entry : timelineCallbacks.entrySet()) { + entry.getValue().execute( + () -> { + if (ServerView.CallbackAction.UNREGISTER == function.apply(entry.getKey())) { + timelineCallbacks.remove(entry.getKey()); + } + } + ); + } + } + + public QueryRunner getQueryRunner(String serverName) + { + return serverQueryRunners.get(serverName); + } + public VersionedIntervalTimeline getTimeline(DataSource dataSource) { String table = Iterables.getOnlyElement(dataSource.getTableNames()); @@ -211,7 +277,6 @@ public VersionedIntervalTimeline getTimeline(DataSource } } - @Override public Map getLoadInfoForAllSegments() { return segmentLoadInfos; diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java b/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java deleted file mode 100644 index b5041ef8e094..000000000000 --- a/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.client; - -import org.apache.druid.query.DataSource; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.VersionedIntervalTimeline; - -import java.util.Map; - -/** - * Segment timeline maintained in the coordinator. It provides methods for retrieving timeline information - * related to datasources and {@link SegmentLoadInfo}. - */ -public interface CoordinatorTimeline extends InventoryView -{ - /** - * Retrieve timeline for a datasource. - */ - VersionedIntervalTimeline getTimeline(DataSource dataSource); - - /** - * Server information for all segments in the timeline. - */ - Map getLoadInfoForAllSegments(); -} diff --git a/server/src/main/java/org/apache/druid/client/DirectDruidClientFactory.java b/server/src/main/java/org/apache/druid/client/DirectDruidClientFactory.java new file mode 100644 index 000000000000..23529f2ccacc --- /dev/null +++ b/server/src/main/java/org/apache/druid/client/DirectDruidClientFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import org.apache.druid.guice.annotations.EscalatedClient; +import org.apache.druid.guice.annotations.Smile; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.QueryWatcher; + +/** + * Factory for building {@link DirectDruidClient} + */ +public class DirectDruidClientFactory +{ + private final ServiceEmitter emitter; + private final QueryToolChestWarehouse warehouse; + private final QueryWatcher queryWatcher; + private final ObjectMapper smileMapper; + private final HttpClient httpClient; + + @Inject + public DirectDruidClientFactory( + final ServiceEmitter emitter, + final QueryToolChestWarehouse warehouse, + final QueryWatcher queryWatcher, + final @Smile ObjectMapper smileMapper, + final @EscalatedClient HttpClient httpClient + ) + { + this.emitter = emitter; + this.warehouse = warehouse; + this.queryWatcher = queryWatcher; + this.smileMapper = smileMapper; + this.httpClient = httpClient; + } + + public DirectDruidClient makeDirectClient(DruidServer server) + { + return new DirectDruidClient( + warehouse, + queryWatcher, + smileMapper, + httpClient, + server.getScheme(), + server.getHost(), + emitter + ); + } +} diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java deleted file mode 100644 index a35f218d19ac..000000000000 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.client; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Iterables; -import com.google.inject.Inject; -import org.apache.druid.client.selector.ServerSelector; -import org.apache.druid.client.selector.TierSelectorStrategy; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.guice.annotations.EscalatedClient; -import org.apache.druid.guice.annotations.Smile; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.query.DataSource; -import org.apache.druid.query.QueryToolChestWarehouse; -import org.apache.druid.query.QueryWatcher; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.utils.CollectionUtils; - -import java.util.Collection; -import java.util.Comparator; -import java.util.Map; - -/** - * ServerView of coordinator for the state of segments being loaded in the cluster. - * - *

    This class extends {@link BrokerServerView} and implements {@link CoordinatorTimeline}. - * The main distinction between this class and {@link CoordinatorServerView} is the maintenance of a timeline - * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline.

    - * - *

    A new timeline class (implementing {@link TimelineServerView}) is required for - * {@link org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache}, which will run on the Coordinator.

    - */ -@ManageLifecycle -public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline -{ - @Inject - public QueryableCoordinatorServerView( - final QueryToolChestWarehouse warehouse, - final QueryWatcher queryWatcher, - final @Smile ObjectMapper smileMapper, - final @EscalatedClient HttpClient httpClient, - final FilteredServerInventoryView baseView, - final TierSelectorStrategy tierSelectorStrategy, - final ServiceEmitter emitter, - final CoordinatorSegmentWatcherConfig segmentWatcherConfig - ) - { - super(warehouse, queryWatcher, smileMapper, httpClient, baseView, tierSelectorStrategy, emitter, new BrokerSegmentWatcherConfig() { - @Override - public boolean isAwaitInitializationOnStart() - { - return segmentWatcherConfig.isAwaitInitializationOnStart(); - } - }); - } - - /** - * Since this class maintains a timeline of {@link ServerSelector} objects, - * this method converts and returns a new timeline of the object {@link SegmentLoadInfo}. - * - * @param dataSource dataSoruce - * @return timeline for the given dataSource - */ - @Override - public VersionedIntervalTimeline getTimeline(DataSource dataSource) - { - String table = Iterables.getOnlyElement(dataSource.getTableNames()); - VersionedIntervalTimeline baseTimeline; - - synchronized (lock) { - baseTimeline = timelines.get(table); - } - - VersionedIntervalTimeline segmentLoadInfoTimeline = - new VersionedIntervalTimeline<>(Comparator.naturalOrder()); - segmentLoadInfoTimeline.addAll( - baseTimeline.iterateAllObjects().stream() - .map(serverSelector -> new VersionedIntervalTimeline.PartitionChunkEntry<>( - serverSelector.getSegment().getInterval(), - serverSelector.getSegment().getVersion(), - serverSelector.getSegment().getShardSpec().createChunk(serverSelector.toSegmentLoadInfo()) - )).iterator()); - - return segmentLoadInfoTimeline; - } - - @Override - public Map getLoadInfoForAllSegments() - { - return CollectionUtils.mapValues(selectors, ServerSelector::toSegmentLoadInfo); - } - - @Override - public DruidServer getInventoryValue(String serverKey) - { - return baseView.getInventoryValue(serverKey); - } - - @Override - public Collection getInventory() - { - return baseView.getInventory(); - } - - @Override - public boolean isStarted() - { - return baseView.isStarted(); - } - - @Override - public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) - { - return baseView.isSegmentLoadedByServer(serverKey, segment); - } -} diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index f9f4c782cc8d..2db810d6313e 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -20,12 +20,14 @@ package org.apache.druid.client; import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; import com.google.common.collect.Sets; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.Overshadowable; import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; public class SegmentLoadInfo implements Overshadowable { @@ -59,6 +61,15 @@ public ImmutableSegmentLoadInfo toImmutableSegmentLoadInfo() return new ImmutableSegmentLoadInfo(segment, servers); } + /** + * Randomly return one server from the sets of {@code servers} + */ + public DruidServerMetadata pickOne() + { + synchronized (this) { + return Iterators.get(servers.iterator(), ThreadLocalRandom.current().nextInt(servers.size())); + } + } @Override public boolean equals(Object o) diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 616778987428..30b259b66194 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -21,7 +21,6 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap; import org.apache.druid.client.DataSegmentInterner; -import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunner; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -216,22 +215,4 @@ public boolean hasData() { return segment.get().hasData(); } - - /** - * This conversion, allows {@link org.apache.druid.client.QueryableCoordinatorServerView} - * to implement methods from {@link org.apache.druid.client.CoordinatorTimeline}. - * - * @return {@link SegmentLoadInfo} - */ - public SegmentLoadInfo toSegmentLoadInfo() - { - List allServers = getAllServers(); - SegmentLoadInfo segmentLoadInfo = new SegmentLoadInfo(segment.get()); - - for (DruidServerMetadata druidServerMetadata : allServers) { - segmentLoadInfo.addServer(druidServerMetadata); - } - - return segmentLoadInfo; - } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index 1095e5473c68..56eda8000199 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -33,8 +33,6 @@ import com.google.common.collect.Maps; import com.google.errorprone.annotations.concurrent.GuardedBy; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.client.ServerView; -import org.apache.druid.client.TimelineServerView; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; @@ -118,7 +116,7 @@ public abstract class AbstractSegmentMetadataCache dataSourceSegments = segmentMetadataInfo .computeIfAbsent( @@ -994,7 +946,7 @@ void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegme * It must be used only in unit tests. */ @VisibleForTesting - void doInLock(Runnable runnable) + protected void doInLock(Runnable runnable) { synchronized (lock) { runnable.run(); diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index ff57ade02429..158ec92031ad 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -21,22 +21,26 @@ import com.google.common.collect.Sets; import com.google.inject.Inject; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.ServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.security.Escalator; +import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import java.io.IOException; import java.util.Set; /** - * Coordinator-side cache of segment metadata that combines segments to identify - * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. + * Coordinator-side cache of segment metadata that combines segments to build + * datasources. The cache provides metadata about a datasource, see {@link DataSourceInformation}. */ @ManageLifecycle public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCache @@ -46,14 +50,60 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach @Inject public CoordinatorSegmentMetadataCache( QueryLifecycleFactory queryLifecycleFactory, - TimelineServerView serverView, + CoordinatorServerView serverView, SegmentMetadataCacheConfig config, Escalator escalator, InternalQueryConfig internalQueryConfig, ServiceEmitter emitter ) { - super(queryLifecycleFactory, serverView, config, escalator, internalQueryConfig, emitter); + super(queryLifecycleFactory, config, escalator, internalQueryConfig, emitter); + + initServerViewTimelineCallback(serverView); + } + + private void initServerViewTimelineCallback(final CoordinatorServerView serverView) + { + serverView.registerTimelineCallback( + callbackExec, + new TimelineServerView.TimelineCallback() + { + @Override + public ServerView.CallbackAction timelineInitialized() + { + synchronized (lock) { + isServerViewInitialized = true; + lock.notifyAll(); + } + + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentAdded(final DruidServerMetadata server, final DataSegment segment) + { + addSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DataSegment segment) + { + removeSegment(segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction serverSegmentRemoved( + final DruidServerMetadata server, + final DataSegment segment + ) + { + removeServerSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + } + ); } /** diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalker.java new file mode 100644 index 000000000000..55a5a9b3fcc1 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalker.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.google.common.collect.Sets; +import com.google.inject.Inject; +import org.apache.druid.client.CachingClusteredClient; +import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.SegmentLoadInfo; +import org.apache.druid.guice.http.DruidHttpClientConfig; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.query.FluentQueryRunner; +import org.apache.druid.query.Queries; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.SegmentDescriptor; +import org.apache.druid.query.context.ResponseContext; +import org.apache.druid.server.SetAndVerifyContextQueryRunner; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.initialization.ServerConfig; +import org.apache.druid.timeline.TimelineLookup; +import org.apache.druid.timeline.TimelineObjectHolder; +import org.apache.druid.timeline.partition.PartitionChunk; +import org.joda.time.Interval; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.function.Function; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +/** + * This {@link QuerySegmentWalker} implementation is specific to SegmentMetadata queries + * executed by {@link CoordinatorSegmentMetadataCache} and is in parity with {@link CachingClusteredClient}. + */ +public class SegmentMetadataQuerySegmentWalker implements QuerySegmentWalker +{ + private static final EmittingLogger log = new EmittingLogger(SegmentMetadataQuerySegmentWalker.class); + private final CoordinatorServerView serverView; + private final DruidHttpClientConfig httpClientConfig; + private final QueryToolChestWarehouse warehouse; + private final ServerConfig serverConfig; + private final ServiceEmitter emitter; + + @Inject + public SegmentMetadataQuerySegmentWalker( + final CoordinatorServerView serverView, + final DruidHttpClientConfig httpClientConfig, + final QueryToolChestWarehouse warehouse, + final ServerConfig serverConfig, + final ServiceEmitter emitter + ) + { + this.serverView = serverView; + this.httpClientConfig = httpClientConfig; + this.warehouse = warehouse; + this.emitter = emitter; + this.serverConfig = serverConfig; + } + + @Override + public QueryRunner getQueryRunnerForIntervals(Query query, Iterable intervals) + { + throw new UnsupportedOperationException(); + } + + @Override + public QueryRunner getQueryRunnerForSegments(Query query, Iterable specs) + { + return decorateRunner(query, new QueryRunner() + { + @Override + public Sequence run(final QueryPlus queryPlus, final ResponseContext responseContext) + { + return SegmentMetadataQuerySegmentWalker.this.run( + queryPlus, + responseContext, + new CachingClusteredClient.TimelineConverter<>(specs) + ); + } + }); + } + + private QueryRunner decorateRunner(Query query, QueryRunner baseClusterRunner) + { + final QueryToolChest> toolChest = warehouse.getToolChest(query); + + final SetAndVerifyContextQueryRunner baseRunner = new SetAndVerifyContextQueryRunner<>( + serverConfig, + baseClusterRunner + ); + + return FluentQueryRunner + .create(baseRunner, toolChest) + .emitCPUTimeMetric(emitter); + } + + private Sequence run( + QueryPlus queryPlus, + final ResponseContext responseContext, + final UnaryOperator> timelineConverter + ) + { + final Query query = queryPlus.getQuery(); + + final TimelineLookup timeline = serverView.getTimeline( + query.getDataSource() + ); + if (null == timeline) { + return Sequences.empty(); + } + + final TimelineLookup timelineLookup = timelineConverter.apply(timeline); + + QueryToolChest> toolChest = warehouse.getToolChest(query); + Set> segmentAndServers = computeSegmentsToQuery(timelineLookup, query, toolChest); + + queryPlus = queryPlus.withQueryMetrics(toolChest); + queryPlus.getQueryMetrics().reportQueriedSegmentCount(segmentAndServers.size()).emit(emitter); + + final SortedMap> serverSegments = groupSegmentsByServer(segmentAndServers, query); + + final List> listOfSequences = new ArrayList<>(serverSegments.size()); + + QueryPlus finalQueryPlus = queryPlus; + serverSegments.forEach((server, segmentsOfServer) -> { + final QueryRunner serverRunner = serverView.getQueryRunner(server); + + if (serverRunner == null) { + log.error("Server [%s] doesn't have a query runner", server); + return; + } + + // Divide user-provided maxQueuedBytes by the number of servers, and limit each server to that much. + final long maxQueuedBytes = httpClientConfig.getMaxQueuedBytes(); + final long maxQueuedBytesPerServer = maxQueuedBytes / serverSegments.size(); + final Sequence serverResults = getServerResults( + serverRunner, + finalQueryPlus, + responseContext, + maxQueuedBytesPerServer, + segmentsOfServer + ); + listOfSequences.add(serverResults); + }); + + return merge(queryPlus.getQuery(), listOfSequences); + } + + Sequence getServerResults( + QueryRunner serverRunner, + QueryPlus queryPlus, + ResponseContext responseContext, + long maxQueuedBytesPerServer, + List segmentDescriptors) + { + return serverRunner.run( + queryPlus.withQuery( + Queries.withSpecificSegments( + queryPlus.getQuery(), + segmentDescriptors + ) + ).withMaxQueuedBytes(maxQueuedBytesPerServer), + responseContext + ); + } + + private Set> computeSegmentsToQuery( + TimelineLookup timeline, + Query query, + QueryToolChest> toolChest + ) + { + final Function>> lookupFn + = timeline::lookup; + + final List intervals = query.getIntervals(); + List> timelineObjectHolders = + intervals.stream().flatMap(i -> lookupFn.apply(i).stream()).collect(Collectors.toList()); + + final List> serversLookup = toolChest.filterSegments(query, timelineObjectHolders); + + Set> segmentAndServers = new HashSet<>(); + for (TimelineObjectHolder holder : serversLookup) { + final Set> filteredChunks = Sets.newHashSet(holder.getObject()); + for (PartitionChunk chunk : filteredChunks) { + SegmentLoadInfo server = chunk.getObject(); + final SegmentDescriptor segment = new SegmentDescriptor( + holder.getInterval(), + holder.getVersion(), + chunk.getChunkNumber() + ); + segmentAndServers.add(new Pair<>(segment, server)); + } + } + + return segmentAndServers; + } + + private SortedMap> groupSegmentsByServer( + Set> segmentAndServers, + Query query + ) + { + final SortedMap> serverSegments = new TreeMap<>(); + + for (Pair segmentAndServer : segmentAndServers) { + final DruidServerMetadata druidServerMetadata = segmentAndServer.rhs.pickOne(); + + if (druidServerMetadata == null) { + log.makeAlert( + "No servers found for SegmentDescriptor[%s] for DataSource[%s]?! How can this be?!", + segmentAndServer.lhs, + query.getDataSource() + ).emit(); + } else { + serverSegments.computeIfAbsent(druidServerMetadata.getName(), s -> new ArrayList<>()).add(segmentAndServer.lhs); + } + } + + return serverSegments; + } + + private Sequence merge(Query query, List> sequencesByInterval) + { + return Sequences + .simple(sequencesByInterval) + .flatMerge(seq -> seq, query.getResultOrdering()); + } +} diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index dfbd559c1b06..277670add7e2 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,7 +30,7 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; -import org.apache.druid.client.CoordinatorTimeline; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -102,7 +102,7 @@ public class DataSourcesResource private static final Logger log = new Logger(DataSourcesResource.class); private static final long DEFAULT_LOADSTATUS_INTERVAL_OFFSET = 14 * 24 * 60 * 60 * 1000; - private final CoordinatorTimeline serverInventoryView; + private final CoordinatorServerView serverInventoryView; private final SegmentsMetadataManager segmentsMetadataManager; private final MetadataRuleManager metadataRuleManager; private final OverlordClient overlordClient; @@ -111,7 +111,7 @@ public class DataSourcesResource @Inject public DataSourcesResource( - CoordinatorTimeline coordinatorTimeline, + CoordinatorServerView serverInventoryView, SegmentsMetadataManager segmentsMetadataManager, MetadataRuleManager metadataRuleManager, @Nullable OverlordClient overlordClient, @@ -119,7 +119,7 @@ public DataSourcesResource( DruidCoordinator coordinator ) { - this.serverInventoryView = coordinatorTimeline; + this.serverInventoryView = serverInventoryView; this.segmentsMetadataManager = segmentsMetadataManager; this.metadataRuleManager = metadataRuleManager; this.overlordClient = overlordClient; diff --git a/server/src/test/java/org/apache/druid/client/BrokerServerViewTest.java b/server/src/test/java/org/apache/druid/client/BrokerServerViewTest.java index cef8328c8b15..509f00a77bdb 100644 --- a/server/src/test/java/org/apache/druid/client/BrokerServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/BrokerServerViewTest.java @@ -637,11 +637,16 @@ public CallbackAction segmentViewInitialized() } }; - brokerServerView = new BrokerServerView( + DirectDruidClientFactory druidClientFactory = new DirectDruidClientFactory( + new NoopServiceEmitter(), EasyMock.createMock(QueryToolChestWarehouse.class), EasyMock.createMock(QueryWatcher.class), getSmileMapper(), - EasyMock.createMock(HttpClient.class), + EasyMock.createMock(HttpClient.class) + ); + + brokerServerView = new BrokerServerView( + druidClientFactory, baseView, new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), new NoopServiceEmitter(), diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java index 7755ae2c122c..6d3c33753d10 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java @@ -267,12 +267,12 @@ public class CachingClusteredClientTest private Cache cache; private DruidServer[] servers; - public CachingClusteredClientTest(int randomSeed) + public CachingClusteredClientTest() { - this.random = new Random(randomSeed); + this.random = new Random(10); } - @Parameterized.Parameters(name = "{0}") + //@Parameterized.Parameters(name = "{0}") public static Iterable constructorFeeder() { return Lists.transform( @@ -2746,7 +2746,7 @@ public int getDefaultMaxQueryParallelism() ); } - private static class ServerExpectation + public static class ServerExpectation { private final SegmentId segmentId; private final Interval interval; @@ -2949,7 +2949,7 @@ && getEndRootPartitionId() >= other.getEndRootPartitionId() } } - private static class ServerExpectations implements Iterable + public static class ServerExpectations implements Iterable { private final DruidServer server; private final QueryRunner queryRunner; diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java similarity index 69% rename from server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java index 402113e051d4..26f3447ec0f6 100644 --- a/server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java @@ -27,14 +27,10 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import org.apache.curator.utils.ZKPaths; -import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; -import org.apache.druid.client.selector.RandomServerSelectorStrategy; import org.apache.druid.curator.CuratorTestBase; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.query.QueryToolChestWarehouse; -import org.apache.druid.query.QueryWatcher; +import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.TestHelper; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -52,87 +48,74 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; -public class CoordinatorTimelineTest extends CuratorTestBase +@RunWith(Parameterized.class) +public class CoordinatorServerViewTest extends CuratorTestBase { - private final ObjectMapper jsonMapper; - private final ZkPathsConfig zkPathsConfig; - private final String inventoryPath; + private ObjectMapper jsonMapper; + private ZkPathsConfig zkPathsConfig; + private String inventoryPath; + private CountDownLatch segmentViewInitLatch; private CountDownLatch segmentAddedLatch; private CountDownLatch segmentRemovedLatch; + private CountDownLatch callbackSegmentViewInitLatch; + private CountDownLatch callbackSegmentAddedLatch; + private CountDownLatch callbackSegmentRemovedLatch; + private CountDownLatch callbackServerSegmentRemovedLatch; + private BatchServerInventoryView baseView; + private CoordinatorServerView coordinatorServerView; + private ExecutorService callbackExec; - public CoordinatorTimelineTest() + private boolean setDruidClientFactory; + + @Parameterized.Parameters + public static Object[] data() { - jsonMapper = TestHelper.makeJsonMapper(); - zkPathsConfig = new ZkPathsConfig(); - inventoryPath = zkPathsConfig.getLiveSegmentsPath(); + return new Object[]{true, false}; + } + + public CoordinatorServerViewTest(boolean setDruidClientFactory) + { + this.setDruidClientFactory = setDruidClientFactory; } @Before public void setUp() throws Exception { + jsonMapper = TestHelper.makeJsonMapper(); + zkPathsConfig = new ZkPathsConfig(); + inventoryPath = zkPathsConfig.getLiveSegmentsPath(); + callbackExec = Execs.singleThreaded("CoordinatorServerViewTest-%s"); + setupServerAndCurator(); curator.start(); curator.blockUntilConnected(); } @Test - public void testSingleServerAddedRemovedSegment_CoordinatorServerView() throws Exception + public void testSingleServerAddedRemovedSegment() throws Exception { segmentViewInitLatch = new CountDownLatch(1); segmentAddedLatch = new CountDownLatch(1); segmentRemovedLatch = new CountDownLatch(1); + callbackSegmentViewInitLatch = new CountDownLatch(1); + callbackSegmentAddedLatch = new CountDownLatch(1); + callbackServerSegmentRemovedLatch = new CountDownLatch(1); + callbackSegmentRemovedLatch = new CountDownLatch(1); - setupViews(); + setupViews(setDruidClientFactory); - CoordinatorServerView coordinatorServerView = new CoordinatorServerView( - baseView, - new CoordinatorSegmentWatcherConfig(), - new NoopServiceEmitter() - ); - - baseView.start(); - coordinatorServerView.start(); - - testSingleServerAddedRemovedSegment(coordinatorServerView); - } - - @Test - public void testSingleServerAddedRemovedSegment_QueryableCoordinatorServerView() throws Exception - { - segmentViewInitLatch = new CountDownLatch(1); - segmentAddedLatch = new CountDownLatch(1); - segmentRemovedLatch = new CountDownLatch(1); - - setupViews(); - - QueryableCoordinatorServerView queryableCoordinatorServerView = new QueryableCoordinatorServerView( - EasyMock.createMock(QueryToolChestWarehouse.class), - EasyMock.createMock(QueryWatcher.class), - new ObjectMapper(), - EasyMock.createMock(HttpClient.class), - baseView, - new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), - new NoopServiceEmitter(), - new CoordinatorSegmentWatcherConfig() - ); - - baseView.start(); - queryableCoordinatorServerView.start(); - - testSingleServerAddedRemovedSegment(queryableCoordinatorServerView); - } - - private void testSingleServerAddedRemovedSegment(CoordinatorTimeline coordinatorTimeline) throws Exception - { final DruidServer druidServer = new DruidServer( "localhost:1234", "localhost:1234", @@ -152,7 +135,16 @@ private void testSingleServerAddedRemovedSegment(CoordinatorTimeline coordinator Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - TimelineLookup timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackSegmentViewInitLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackSegmentAddedLatch)); + + if (setDruidClientFactory) { + Assert.assertNotNull(coordinatorServerView.getQueryRunner(druidServer.getName())); + } else { + Assert.assertNull(coordinatorServerView.getQueryRunner(druidServer.getName())); + } + + TimelineLookup timeline = coordinatorServerView.getTimeline(new TableDataSource("test_overlord_server_view")); List serverLookupRes = (List) timeline.lookup( intervals ); @@ -176,8 +168,9 @@ private void testSingleServerAddedRemovedSegment(CoordinatorTimeline coordinator unannounceSegmentForServer(druidServer, segment); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackServerSegmentRemovedLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackSegmentRemovedLatch)); - timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() @@ -186,54 +179,21 @@ private void testSingleServerAddedRemovedSegment(CoordinatorTimeline coordinator } @Test - public void testMultipleServerAddedRemovedSegment_CoordinatorServerView() throws Exception + public void testMultipleServerAddedRemovedSegment() throws Exception { segmentViewInitLatch = new CountDownLatch(1); segmentAddedLatch = new CountDownLatch(5); - segmentRemovedLatch = new CountDownLatch(1); - - setupViews(); - - CoordinatorServerView coordinatorServerView = new CoordinatorServerView( - baseView, - new CoordinatorSegmentWatcherConfig(), - new NoopServiceEmitter() - ); - - baseView.start(); - coordinatorServerView.start(); - - testMultipleServerAddedRemovedSegment(coordinatorServerView); - } - @Test - public void testMultipleServerAddedRemovedSegment_QueryableCoordinatorServerView() throws Exception - { - segmentViewInitLatch = new CountDownLatch(1); - segmentAddedLatch = new CountDownLatch(5); + // temporarily set latch count to 1 segmentRemovedLatch = new CountDownLatch(1); - setupViews(); + callbackSegmentViewInitLatch = new CountDownLatch(1); + callbackSegmentAddedLatch = new CountDownLatch(5); + callbackServerSegmentRemovedLatch = new CountDownLatch(1); + callbackSegmentRemovedLatch = new CountDownLatch(1); - QueryableCoordinatorServerView queryableCoordinatorServerView = new QueryableCoordinatorServerView( - EasyMock.createMock(QueryToolChestWarehouse.class), - EasyMock.createMock(QueryWatcher.class), - new ObjectMapper(), - EasyMock.createMock(HttpClient.class), - baseView, - new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), - new NoopServiceEmitter(), - new CoordinatorSegmentWatcherConfig() - ); - - baseView.start(); - queryableCoordinatorServerView.start(); + setupViews(setDruidClientFactory); - testMultipleServerAddedRemovedSegment(queryableCoordinatorServerView); - } - - void testMultipleServerAddedRemovedSegment(CoordinatorTimeline coordinatorTimeline) throws Exception - { final List druidServers = Lists.transform( ImmutableList.of("localhost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), new Function() @@ -280,8 +240,18 @@ public DataSegment apply(Pair input) } Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackSegmentViewInitLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackSegmentAddedLatch)); - TimelineLookup timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); + for (int i = 0; i < 5; ++i) { + if (setDruidClientFactory) { + Assert.assertNotNull(coordinatorServerView.getQueryRunner(druidServers.get(i).getName())); + } else { + Assert.assertNull(coordinatorServerView.getQueryRunner(druidServers.get(i).getName())); + } + } + + TimelineLookup timeline = coordinatorServerView.getTimeline(new TableDataSource("test_overlord_server_view")); assertValues( Arrays.asList( createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), @@ -298,12 +268,15 @@ public DataSegment apply(Pair input) // unannounce the segment created by dataSegmentWithIntervalAndVersion("2011-04-01/2011-04-09", "v2") unannounceSegmentForServer(druidServers.get(2), segments.get(2)); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackSegmentRemovedLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackServerSegmentRemovedLatch)); // renew segmentRemovedLatch since we still have 4 segments to unannounce segmentRemovedLatch = new CountDownLatch(4); + callbackServerSegmentRemovedLatch = new CountDownLatch(4); + callbackSegmentRemovedLatch = new CountDownLatch(4); - - timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); + timeline = coordinatorServerView.getTimeline(new TableDataSource("test_overlord_server_view")); assertValues( Arrays.asList( createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), @@ -322,8 +295,9 @@ public DataSegment apply(Pair input) } } Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackSegmentRemovedLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(callbackServerSegmentRemovedLatch)); - timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() @@ -369,10 +343,11 @@ private void assertValues( Assert.assertFalse(segmentLoadInfo.isEmpty()); Assert.assertEquals(expectedPair.rhs.rhs.lhs.getMetadata(), Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers())); + Assert.assertEquals(expectedPair.rhs.rhs.lhs.getMetadata(), segmentLoadInfo.pickOne()); } } - private void setupViews() + private void setupViews(boolean setDruidClientFactory) throws Exception { baseView = new BatchServerInventoryView( zkPathsConfig, @@ -416,6 +391,69 @@ public CallbackAction segmentViewInitialized() ); } }; + + DirectDruidClientFactory druidClientFactory = null; + + if (setDruidClientFactory) { + druidClientFactory = EasyMock.createMock(DirectDruidClientFactory.class); + DirectDruidClient directDruidClient = EasyMock.mock(DirectDruidClient.class); + EasyMock.expect(druidClientFactory.makeDirectClient(EasyMock.anyObject(DruidServer.class))) + .andReturn(directDruidClient) + .anyTimes(); + + EasyMock.replay(druidClientFactory); + } + + coordinatorServerView = new CoordinatorServerView( + baseView, + new CoordinatorSegmentWatcherConfig(), + new NoopServiceEmitter(), + druidClientFactory + ); + + baseView.start(); + initServerViewTimelineCallback(coordinatorServerView); + coordinatorServerView.start(); + } + + private void initServerViewTimelineCallback(final CoordinatorServerView serverView) + { + serverView.registerTimelineCallback( + callbackExec, + new TimelineServerView.TimelineCallback() + { + @Override + public ServerView.CallbackAction timelineInitialized() + { + callbackSegmentViewInitLatch.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentAdded(final DruidServerMetadata server, final DataSegment segment) + { + callbackSegmentAddedLatch.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DataSegment segment) + { + callbackSegmentRemovedLatch.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction serverSegmentRemoved( + final DruidServerMetadata server, + final DataSegment segment + ) + { + callbackServerSegmentRemovedLatch.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + } + ); } private DataSegment dataSegmentWithIntervalAndVersion(String intervalStr, String version) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java similarity index 86% rename from server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java index a26b82722f86..fd4696c315f9 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java @@ -19,29 +19,22 @@ package org.apache.druid.segment.metadata; -import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; -import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; +import org.apache.druid.client.CoordinatorSegmentWatcherConfig; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidServer; -import org.apache.druid.client.FilteredServerInventoryView; -import org.apache.druid.client.FilteringSegmentCallback; import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.ServerInventoryView; +import org.apache.druid.client.ServerView; import org.apache.druid.client.ServerView.CallbackAction; -import org.apache.druid.client.ServerView.SegmentCallback; -import org.apache.druid.client.ServerView.ServerRemovedCallback; +import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.TimelineServerView.TimelineCallback; -import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; -import org.apache.druid.client.selector.RandomServerSelectorStrategy; -import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.guice.http.DruidHttpClientConfig; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.NonnullPair; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.query.QueryToolChestWarehouse; -import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -50,9 +43,9 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.timeline.DataSegment; @@ -83,26 +76,76 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -public class SegmentDataCacheConcurrencyTest extends SegmentMetadataCacheCommon +public class CoordinatorSegmentDataCacheConcurrencyTest extends SegmentMetadataCacheCommon { private static final String DATASOURCE = "datasource"; static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); private File tmpDir; private TestServerInventoryView inventoryView; - private BrokerServerView serverView; + private CoordinatorServerView serverView; private AbstractSegmentMetadataCache schema; private ExecutorService exec; + private TestSegmentMetadataQueryWalker walker; @Before public void setUp() throws Exception { + setUpData(); setUpCommon(); tmpDir = temporaryFolder.newFolder(); - walker = new SpecificSegmentsQuerySegmentWalker(conglomerate); inventoryView = new TestServerInventoryView(); - serverView = newBrokerServerView(inventoryView); + serverView = newCoordinatorServerView(inventoryView); + walker = new TestSegmentMetadataQueryWalker( + serverView, + new DruidHttpClientConfig() + { + @Override + public long getMaxQueuedBytes() + { + return 0L; + } + }, + queryToolChestWarehouse, + new ServerConfig(), + new NoopServiceEmitter(), + conglomerate, + new HashMap<>() + ); + + CountDownLatch initLatch = new CountDownLatch(1); + serverView.registerTimelineCallback( + Execs.singleThreaded("ServerViewInit-DruidSchemaConcurrencyTest-%d"), + new TimelineServerView.TimelineCallback() + { + @Override + public ServerView.CallbackAction timelineInitialized() + { + initLatch.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + return null; + } + + @Override + public CallbackAction segmentRemoved(DataSegment segment) + { + return null; + } + + @Override + public CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegment segment) + { + return null; + } + } + ); + inventoryView.init(); - serverView.awaitInitialization(); + initLatch.await(); exec = Execs.multiThreaded(4, "DruidSchemaConcurrencyTest-%d"); } @@ -112,7 +155,6 @@ public void tearDown() throws Exception { super.tearDown(); exec.shutdownNow(); - walker.close(); } /** @@ -212,10 +254,8 @@ public CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegme Assert.assertFalse(refreshFuture.isDone()); for (int i = 0; i < 1000; i++) { - boolean hasTimeline = exec.submit( - () -> serverView.getTimeline((new TableDataSource(DATASOURCE)).getAnalysis()) - .isPresent() - ).get(100, TimeUnit.MILLISECONDS); + boolean hasTimeline = exec.submit(() -> (serverView.getTimeline(new TableDataSource(DATASOURCE)) != null)) + .get(100, TimeUnit.MILLISECONDS); Assert.assertTrue(hasTimeline); // We want to call getTimeline while BrokerServerView is being updated. Sleep might help with timing. Thread.sleep(2); @@ -329,9 +369,9 @@ private void addSegmentsToCluster(int partitionIdStart, int numServers, int numS for (int i = 0; i < numSegments; i++) { DataSegment segment = newSegment(i + partitionIdStart); QueryableIndex index = newQueryableIndex(i + partitionIdStart); - walker.add(segment, index); int serverIndex = i % numServers; inventoryView.addServerSegment(newServer("server_" + serverIndex), segment); + walker.add(segment, index); } } @@ -354,17 +394,13 @@ private void removeSegmentsFromCluster(int numServers, int numSegments) } } - private static BrokerServerView newBrokerServerView(FilteredServerInventoryView baseView) + private static CoordinatorServerView newCoordinatorServerView(ServerInventoryView baseView) { - return new BrokerServerView( - EasyMock.createMock(QueryToolChestWarehouse.class), - EasyMock.createMock(QueryWatcher.class), - new DefaultObjectMapper(), - EasyMock.createMock(HttpClient.class), + return new CoordinatorServerView( baseView, - new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), + EasyMock.createMock(CoordinatorSegmentWatcherConfig.class), new NoopServiceEmitter(), - new BrokerSegmentWatcherConfig() + null ); } @@ -416,7 +452,7 @@ private QueryableIndex newQueryableIndex(int partitionId) .buildMMappedIndex(); } - private static class TestServerInventoryView implements FilteredServerInventoryView + private static class TestServerInventoryView implements ServerInventoryView { private final Map serverMap = new HashMap<>(); private final Map> segmentsMap = new HashMap<>(); @@ -452,11 +488,10 @@ private void removeServer(DruidServer server) @Override public void registerSegmentCallback( Executor exec, - SegmentCallback callback, - Predicate> filter + SegmentCallback callback ) { - segmentCallbacks.add(new NonnullPair<>(new FilteringSegmentCallback(callback, filter), exec)); + segmentCallbacks.add(new NonnullPair<>(callback, exec)); } @Override diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheCommon.java new file mode 100644 index 000000000000..8ce1e7fe2035 --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheCommon.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.google.common.collect.Lists; +import org.apache.druid.client.DruidServer; +import org.apache.druid.guice.http.DruidHttpClientConfig; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.SegmentDescriptor; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.server.initialization.ServerConfig; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.timeline.DataSegment; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CoordinatorSegmentMetadataCacheCommon extends SegmentMetadataCacheCommon +{ + public TestSegmentMetadataQueryWalker walker; + public TestCoordinatorServerView serverView; + public List druidServers; + + public void setUp() throws Exception + { + setUpData(); + setUpCommon(); + + serverView = new TestCoordinatorServerView( + Lists.newArrayList(segment1, segment2, segment3, segment4, segment5), + Lists.newArrayList(realtimeSegment1) + ); + + Map> queryableIndexMap = new HashMap<>(); + queryableIndexMap.put(segment1.toDescriptor(), Pair.of(index1, segment1)); + queryableIndexMap.put(segment2.toDescriptor(), Pair.of(index2, segment2)); + queryableIndexMap.put(segment3.toDescriptor(), Pair.of(index2, segment3)); + queryableIndexMap.put(segment4.toDescriptor(), Pair.of(indexAuto1, segment4)); + queryableIndexMap.put(segment5.toDescriptor(), Pair.of(indexAuto2, segment5)); + + walker = new TestSegmentMetadataQueryWalker( + serverView, + new DruidHttpClientConfig() + { + @Override + public long getMaxQueuedBytes() + { + return 0L; + } + }, + queryToolChestWarehouse, + new ServerConfig(), + new NoopServiceEmitter(), + conglomerate, + queryableIndexMap + ); + + druidServers = serverView.getInventory(); + } +} diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java index 90664e7c87d0..26171366d5ac 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java @@ -25,20 +25,31 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.client.DruidServer; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; +import org.apache.druid.query.DruidMetrics; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.QueryResponse; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; @@ -49,6 +60,7 @@ import org.junit.Test; import java.io.IOException; +import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -58,7 +70,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class CoordinatorSegmentMetadataCacheTest extends SegmentMetadataCacheCommon +public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetadataCacheCommon { // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); @@ -68,10 +80,10 @@ public class CoordinatorSegmentMetadataCacheTest extends SegmentMetadataCacheCom private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @Before - public void setup() throws Exception + @Override + public void setUp() throws Exception { - setUpCommon(); - setupData(); + super.setUp(); } @After @@ -82,7 +94,6 @@ public void tearDown() throws Exception if (runningSchema != null) { runningSchema.stop(); } - walker.close(); } public CoordinatorSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException @@ -277,21 +288,101 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt public void testAvailableSegmentMetadataNumRows() throws InterruptedException { CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkAvailableSegmentMetadataNumRows(schema); + + Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); + final List segments = segmentsMetadata.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // find the only segment with datasource "foo2" + final DataSegment existingSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(existingSegment); + final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); + // update AvailableSegmentMetadata of existingSegment with numRows=5 + AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); + schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); + + // find a druidServer holding existingSegment + final Pair pair = druidServers + .stream() + .flatMap(druidServer -> + serverView.getSegmentsOfServer(druidServer).stream() + .filter(segment -> segment.getId().equals(existingSegment.getId())) + .map(segment -> Pair.of(druidServer, segment)) + ) + .findAny() + .orElse(null); + + Assert.assertNotNull(pair); + final DruidServer server = pair.lhs; + Assert.assertNotNull(server); + final DruidServerMetadata druidServerMetadata = server.getMetadata(); + // invoke SegmentMetadataCache#addSegment on existingSegment + schema.addSegment(druidServerMetadata, existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + // get the only segment with datasource "foo2" + final DataSegment currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); + Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); + // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before + Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); } @Test public void testNullDatasource() throws IOException, InterruptedException { CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkNullDatasource(schema); + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // segments contains two segments with datasource "foo" and one with datasource "foo2" + // let's remove the only segment with datasource "foo2" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkNullAvailableSegmentMetadata(schema); + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // remove one of the segments with datasource "foo" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } @Test @@ -312,7 +403,7 @@ public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException final AvailableSegmentMetadata metadata = segmentsMetadata.get(existingSegment.getId()); Assert.assertEquals(1L, metadata.isRealtime()); // get the historical server - final ImmutableDruidServer historicalServer = druidServers.stream() + final DruidServer historicalServer = druidServers.stream() .filter(s -> s.getType().equals(ServerType.HISTORICAL)) .findAny() .orElse(null); @@ -332,7 +423,7 @@ public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); Assert.assertEquals(0L, currentMetadata.isRealtime()); - ImmutableDruidServer realtimeServer = druidServers.stream() + DruidServer realtimeServer = druidServers.stream() .filter(s -> s.getType().equals(ServerType.REALTIME)) .findAny() .orElse(null); @@ -785,7 +876,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. - CoordinatorSegmentMetadataCache mySchema = new CoordinatorSegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( factoryMock, serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -794,7 +885,39 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception new NoopServiceEmitter() ); - checkRunSegmentMetadataQueryWithContext(mySchema, factoryMock, lifecycleMock); + Map queryContext = ImmutableMap.of( + QueryContexts.PRIORITY_KEY, 5, + QueryContexts.BROKER_PARALLEL_MERGE_KEY, false + ); + + DataSegment segment = newSegment("test", 0); + List segmentIterable = ImmutableList.of(segment.getId()); + + // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. + SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(segment.getDataSource()), + new MultipleSpecificSegmentSpec( + segmentIterable.stream() + .map(SegmentId::toDescriptor).collect(Collectors.toList())), + new AllColumnIncluderator(), + false, + queryContext, + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); + // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context + EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) + .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); + + EasyMock.replay(factoryMock, lifecycleMock); + + schema.runSegmentMetadataQuery(segmentIterable); + + EasyMock.verify(factoryMock, lifecycleMock); } @Test @@ -945,6 +1068,16 @@ public void removeSegment(final DataSegment segment) } }; - checkRefreshShouldEmitMetrics(schema, dataSource, emitter, addSegmentLatch); + List segments = ImmutableList.of( + newSegment(dataSource, 1), + newSegment(dataSource, 2) + ); + serverView.addSegment(segments.get(0), ServerType.HISTORICAL); + serverView.addSegment(segments.get(1), ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(dataSource)); + + emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); + emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); } } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 42f365b9af85..2fc04ba658fd 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -22,79 +22,50 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; -import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.MapInputRowParser; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.common.io.Closer; -import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; -import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.Query; -import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; -import org.apache.druid.query.TableDataSource; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; -import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; -import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; -import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.QueryResponse; import org.apache.druid.server.QueryStackTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; -import org.apache.druid.server.security.Access; -import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; -import org.easymock.EasyMock; -import org.junit.Assert; import org.junit.Rule; import org.junit.rules.TemporaryFolder; import java.io.File; -import java.io.IOException; -import java.util.EnumSet; import java.util.List; import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; public abstract class SegmentMetadataCacheCommon { public static final String DATASOURCE1 = "foo"; public static final String DATASOURCE2 = "foo2"; - public static final String DATASOURCE3 = "numfoo"; - public static final String DATASOURCE4 = "foo4"; - public static final String DATASOURCE5 = "lotsocolumns"; - public static final String BROADCAST_DATASOURCE = "broadcast"; - public static final String FORBIDDEN_DATASOURCE = "forbiddenDatasource"; + public static final String DATASOURCE3 = "foo3"; public static final String SOME_DATASOURCE = "some_datasource"; public static final String TIMESTAMP_COLUMN = "t"; private static final InputRowSchema FOO_SCHEMA = new InputRowSchema( @@ -105,13 +76,13 @@ public abstract class SegmentMetadataCacheCommon null ); - final List ROWS1 = ImmutableList.of( + public final List ROWS1 = ImmutableList.of( createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) ); - final List ROWS2 = ImmutableList.of( + public final List ROWS2 = ImmutableList.of( createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) @@ -120,19 +91,26 @@ public abstract class SegmentMetadataCacheCommon public QueryRunnerFactoryConglomerate conglomerate; public Closer resourceCloser; public QueryToolChestWarehouse queryToolChestWarehouse; + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - public SpecificSegmentsQuerySegmentWalker walker; - public TestTimelineServerView serverView; - public List druidServers; - public QueryableIndex index1; public QueryableIndex index2; public QueryableIndex indexAuto1; public QueryableIndex indexAuto2; + public DataSegment realtimeSegment1; + public DataSegment segment1; + public DataSegment segment2; + public DataSegment segment3; + public DataSegment segment4; + public DataSegment segment5; + + static { + NullHandling.initializeForTests(); + } public void setUpCommon() { @@ -148,7 +126,7 @@ public > QueryToolChest getToolChest }; } - public void setupData() throws Exception + public void setUpData() throws Exception { final File tmpDir = temporaryFolder.newFolder(); index1 = IndexBuilder.create() @@ -251,54 +229,49 @@ public void setupData() throws Exception .rows(autoRows2) .buildMMappedIndex(); - walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( + segment1 = DataSegment.builder() .dataSource(DATASOURCE1) .interval(Intervals.of("2000/P1Y")) .version("1") .shardSpec(new LinearShardSpec(0)) .size(0) - .build(), - index1 - ).add( + .build(); + segment2 = DataSegment.builder() .dataSource(DATASOURCE1) .interval(Intervals.of("2001/P1Y")) .version("1") .shardSpec(new LinearShardSpec(0)) .size(0) - .build(), - index2 - ).add( + .build(); + segment3 = DataSegment.builder() .dataSource(DATASOURCE2) .interval(index2.getDataInterval()) .version("1") .shardSpec(new LinearShardSpec(0)) .size(0) - .build(), - index2 - ).add( + .build(); + segment4 = DataSegment.builder() .dataSource(SOME_DATASOURCE) .interval(Intervals.of("2023-01-01T00Z/P1D")) .version("1") - .shardSpec(new LinearShardSpec(1)) + .shardSpec(new LinearShardSpec(0)) .size(0) - .build(), - indexAuto1 - ).add( + .build(); + segment5 = DataSegment.builder() .dataSource(SOME_DATASOURCE) .interval(Intervals.of("2023-01-02T00Z/P1D")) .version("1") - .shardSpec(new LinearShardSpec(1)) + .shardSpec(new LinearShardSpec(0)) .size(0) - .build(), - indexAuto2 - ); + .build(); + realtimeSegment1 = new DataSegment( - "foo3", + DATASOURCE3, Intervals.of("2012/2013"), "version3", null, @@ -310,10 +283,6 @@ public void setupData() throws Exception 100L, DataSegment.PruneSpecsHolder.DEFAULT ); - - final List realtimeSegments = ImmutableList.of(realtimeSegment1); - serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); - druidServers = serverView.getDruidServers(); } public void tearDown() throws Exception @@ -331,7 +300,7 @@ InputRow createRow(final ImmutableMap map, InputRowSchema inputRowSch return MapInputRowParser.parse(inputRowSchema, (Map) map); } - QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) + public QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) { return new QueryLifecycleFactory( queryToolChestWarehouse, @@ -345,157 +314,6 @@ QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) ); } - public void checkRefreshShouldEmitMetrics( - AbstractSegmentMetadataCache schema, - String dataSource, - StubServiceEmitter emitter, - CountDownLatch addSegmentLatch - ) - throws IOException, InterruptedException - { - List segments = ImmutableList.of( - newSegment(dataSource, 1), - newSegment(dataSource, 2) - ); - serverView.addSegment(segments.get(0), ServerType.HISTORICAL); - serverView.addSegment(segments.get(1), ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(dataSource)); - - emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); - emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); - } - - public void checkNullAvailableSegmentMetadata(AbstractSegmentMetadataCache schema) throws IOException - { - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // remove one of the segments with datasource "foo" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); - } - - public void checkNullDatasource(AbstractSegmentMetadataCache schema) throws IOException - { - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // segments contains two segments with datasource "foo" and one with datasource "foo2" - // let's remove the only segment with datasource "foo2" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); - } - - public void checkAvailableSegmentMetadataNumRows(AbstractSegmentMetadataCache schema) - { - Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); - final List segments = segmentsMetadata.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // find the only segment with datasource "foo2" - final DataSegment existingSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(existingSegment); - final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); - // update AvailableSegmentMetadata of existingSegment with numRows=5 - AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); - schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); - // find a druidServer holding existingSegment - final Pair pair = druidServers - .stream() - .flatMap(druidServer -> druidServer - .iterateAllSegments() - .stream() - .filter(segment -> segment.getId().equals(existingSegment.getId())) - .map(segment -> Pair.of(druidServer, segment)) - ) - .findAny() - .orElse(null); - Assert.assertNotNull(pair); - final ImmutableDruidServer server = pair.lhs; - Assert.assertNotNull(server); - final DruidServerMetadata druidServerMetadata = server.getMetadata(); - // invoke SegmentMetadataCache#addSegment on existingSegment - schema.addSegment(druidServerMetadata, existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - // get the only segment with datasource "foo2" - final DataSegment currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); - Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); - // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before - Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); - } - - public void checkRunSegmentMetadataQueryWithContext(AbstractSegmentMetadataCache schema, QueryLifecycleFactory factoryMock, QueryLifecycle lifecycleMock) - { - Map queryContext = ImmutableMap.of( - QueryContexts.PRIORITY_KEY, 5, - QueryContexts.BROKER_PARALLEL_MERGE_KEY, false - ); - - DataSegment segment = newSegment("test", 0); - List segmentIterable = ImmutableList.of(segment.getId()); - - // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. - SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( - new TableDataSource(segment.getDataSource()), - new MultipleSpecificSegmentSpec( - segmentIterable.stream() - .map(SegmentId::toDescriptor).collect(Collectors.toList())), - new AllColumnIncluderator(), - false, - queryContext, - EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), - false, - null, - null - ); - - EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); - // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context - EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) - .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); - - EasyMock.replay(factoryMock, lifecycleMock); - - schema.runSegmentMetadataQuery(segmentIterable); - - EasyMock.verify(factoryMock, lifecycleMock); - } - public DataSegment newSegment(String datasource, int partitionId) { return new DataSegment( diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java new file mode 100644 index 000000000000..d74a2deb6e80 --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.druid.client.CachingClusteredClientTest.ServerExpectation; +import org.apache.druid.client.CachingClusteredClientTest.ServerExpectations; +import org.apache.druid.client.CoordinatorSegmentWatcherConfig; +import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.DirectDruidClientFactory; +import org.apache.druid.client.DruidServer; +import org.apache.druid.client.SegmentLoadInfo; +import org.apache.druid.client.ServerInventoryView; +import org.apache.druid.guice.http.DruidHttpClientConfig; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.java.util.common.guava.Yielder; +import org.apache.druid.java.util.common.guava.Yielders; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.MapQueryToolChestWarehouse; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.SegmentDescriptor; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.context.ResponseContext; +import org.apache.druid.query.metadata.SegmentMetadataQueryConfig; +import org.apache.druid.query.metadata.SegmentMetadataQueryQueryToolChest; +import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.initialization.ServerConfig; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.timeline.partition.SingleDimensionShardSpec; +import org.easymock.EasyMock; +import org.easymock.IAnswer; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +public class SegmentMetadataQuerySegmentWalkerTest +{ + private final String DATASOURCE = "testDatasource"; + private QueryToolChestWarehouse warehouse; + private DruidHttpClientConfig httpClientConfig; + private DruidServer[] servers; + private Random random; + + @Before + public void setUp() throws Exception + { + warehouse = new MapQueryToolChestWarehouse( + ImmutableMap., QueryToolChest>builder() + .put( + SegmentMetadataQuery.class, + new SegmentMetadataQueryQueryToolChest( + new SegmentMetadataQueryConfig("P1W") + + ) + ).build()); + + httpClientConfig = new DruidHttpClientConfig() + { + @Override + public long getMaxQueuedBytes() + { + return 0L; + } + }; + + servers = + new DruidServer[]{ + new DruidServer("test1", "test1", null, 10, ServerType.HISTORICAL, "bye", 0), + new DruidServer("test2", "test2", null, 10, ServerType.HISTORICAL, "bye", 0), + new DruidServer("test3", "test2", null, 10, ServerType.REALTIME, "bye", 0) + }; + + random = new Random(0); + } + + @Test + public void testWalker() throws IOException + { + Map> timelines = new HashMap<>(); + Map queryRunnerMap = new HashMap<>(); + + Map serverExpectationsMap = + populateTimeline( + timelines, + queryRunnerMap, + Lists.newArrayList( + Pair.of(Intervals.of("2011-01-01/2011-01-02"), 5), + Pair.of(Intervals.of("2011-01-05/2011-01-07"), 1)) + ); + + List segmentDescriptors = + serverExpectationsMap.values() + .stream() + .flatMap(serverExpectations -> Lists.newArrayList(serverExpectations.iterator()).stream()) + .map( + ServerExpectation::getSegment) + .map(segment -> segment.getId().toDescriptor()) + .collect( + Collectors.toList()); + + final SegmentMetadataQuery segmentMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(DATASOURCE), + new MultipleSpecificSegmentSpec( + segmentDescriptors + ), + new AllColumnIncluderator(), + false, + QueryContexts.override( + Collections.emptyMap(), + QueryContexts.BROKER_PARALLEL_MERGE_KEY, + false + ), + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + SegmentMetadataQuerySegmentWalker walker = new SegmentMetadataQuerySegmentWalker( + new TestCoordinatorServerView(timelines, queryRunnerMap), + httpClientConfig, + warehouse, + new ServerConfig(), + new NoopServiceEmitter() + ); + + Sequence resultSequence = walker.getQueryRunnerForSegments( + segmentMetadataQuery, + segmentDescriptors + ).run(QueryPlus.wrap(segmentMetadataQuery)); + + + Yielder yielder = Yielders.each(resultSequence); + Set actualSegmentIds = new HashSet<>(); + try { + while (!yielder.isDone()) { + final SegmentAnalysis analysis = yielder.get(); + actualSegmentIds.add(analysis.getId()); + yielder = yielder.next(null); + } + } + finally { + yielder.close(); + } + + Set expectedSegmentIds = + serverExpectationsMap.values() + .stream() + .flatMap(serverExpectations -> Lists.newArrayList( + serverExpectations.iterator()).stream()) + .map( + ServerExpectation::getSegment) + .map(segment -> segment.getId().toString()) + .collect( + Collectors.toSet()); + Assert.assertEquals(expectedSegmentIds, actualSegmentIds); + } + + private Map populateTimeline( + final Map> timelines, + final Map queryRunnerMap, + final List> intervalAndChunks + ) + { + VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(Comparator.naturalOrder()); + timelines.put(new TableDataSource(DATASOURCE), timeline); + + Map serverExpectationsMap = new HashMap<>(); + + for (Pair intervalAndChunk : intervalAndChunks) { + for (int partitionNum = 0; partitionNum < intervalAndChunk.rhs; partitionNum++) { + + Interval interval = intervalAndChunk.lhs; + int numChunks = intervalAndChunk.rhs; + SegmentId segmentId = SegmentId.of(DATASOURCE, interval, "0", partitionNum); + + DataSegment mockSegment = EasyMock.mock(DataSegment.class); + + final ShardSpec shardSpec; + if (numChunks == 1) { + shardSpec = new SingleDimensionShardSpec("dimAll", null, null, 0, 1); + } else { + String start = null; + String end = null; + if (partitionNum > 0) { + start = String.valueOf(partitionNum); + } + if (partitionNum + 1 < numChunks) { + end = String.valueOf(partitionNum + 1); + } + shardSpec = new SingleDimensionShardSpec("dim", start, end, partitionNum, numChunks); + } + + ServerExpectation expectation = new ServerExpectation<>( + segmentId, + interval, + mockSegment, + shardSpec, + null + ); + DruidServer server = servers[random.nextInt(servers.length)]; + + EasyMock.expect(mockSegment.getShardSpec()) + .andReturn(shardSpec) + .anyTimes(); + + EasyMock.replay(mockSegment); + + serverExpectationsMap.computeIfAbsent( + server.getName(), + s -> new ServerExpectations(server, EasyMock.mock(QueryRunner.class)) + ); + + SegmentLoadInfo segmentLoadInfo = new SegmentLoadInfo(expectation.getSegment()); + segmentLoadInfo.addServer(server.getMetadata()); + serverExpectationsMap.get(server.getName()).addExpectation(expectation); + + queryRunnerMap.computeIfAbsent(server.getName(), v -> serverExpectationsMap.get(server.getName()).getQueryRunner()); + timeline.add(interval, "0", shardSpec.createChunk(segmentLoadInfo)); + timelines.put(new TableDataSource(DATASOURCE), timeline); + } + } + + for (ServerExpectations serverExpectations : serverExpectationsMap.values()) { + QueryRunner queryRunner = serverExpectations.getQueryRunner(); + List serverExpectationList = Lists.newArrayList(serverExpectations.iterator()); + + EasyMock.expect(queryRunner.run(EasyMock.anyObject(QueryPlus.class), EasyMock.anyObject(ResponseContext.class))) + .andReturn(Sequences.simple(toSegmentAnalysis(serverExpectationList))) + .anyTimes(); + EasyMock.replay(queryRunner); + } + return serverExpectationsMap; + } + + private List toSegmentAnalysis(List serverExpectationList) + { + List segmentAnalyses = new ArrayList<>(); + + for (ServerExpectation serverExpectation : serverExpectationList) { + SegmentAnalysis mockSegmentAnalysis = EasyMock.mock(SegmentAnalysis.class); + EasyMock.expect(mockSegmentAnalysis.getId()).andReturn(serverExpectation.getSegmentId().toString()).anyTimes(); + EasyMock.expect(mockSegmentAnalysis.compareTo(EasyMock.isA(SegmentAnalysis.class))) + .andAnswer(new IAnswer() { + @Override + public Integer answer() + { + SegmentAnalysis otherSegment = (SegmentAnalysis) EasyMock.getCurrentArguments()[0]; + String thisId = mockSegmentAnalysis.getId(); + String otherId = otherSegment.getId(); + return thisId.compareTo(otherId); + } + }).anyTimes(); + EasyMock.replay(mockSegmentAnalysis); + segmentAnalyses.add(mockSegmentAnalysis); + } + + return segmentAnalyses; + } + + private static class TestCoordinatorServerView extends CoordinatorServerView + { + private final Map> timelines; + private final Map queryRunnerMap; + + public TestCoordinatorServerView( + final Map> timelines, + final Map queryRunnerMap + ) + { + super( + Mockito.mock(ServerInventoryView.class), + Mockito.mock(CoordinatorSegmentWatcherConfig.class), + Mockito.mock(ServiceEmitter.class), + Mockito.mock(DirectDruidClientFactory.class) + ); + this.timelines = timelines; + this.queryRunnerMap = queryRunnerMap; + } + + @Override + public QueryRunner getQueryRunner(String serverName) + { + return queryRunnerMap.get(serverName); + } + + @Override + public VersionedIntervalTimeline getTimeline(DataSource dataSource) + { + return timelines.get(dataSource); + } + } +} diff --git a/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java b/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java new file mode 100644 index 000000000000..160b8b9c5e09 --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.google.common.collect.Lists; +import org.apache.druid.client.CoordinatorSegmentWatcherConfig; +import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.DirectDruidClientFactory; +import org.apache.druid.client.DruidServer; +import org.apache.druid.client.SegmentLoadInfo; +import org.apache.druid.client.ServerInventoryView; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.SegmentDescriptor; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.timeline.partition.SingleDimensionShardSpec; +import org.easymock.EasyMock; +import org.mockito.Mockito; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executor; + +public class TestCoordinatorServerView extends CoordinatorServerView +{ + private static final DruidServer DUMMY_SERVER = new DruidServer( + "dummy", + "dummy", + null, + 0, + ServerType.HISTORICAL, + "dummy", + 0 + ); + private static final DruidServer DUMMY_SERVER_REALTIME = new DruidServer( + "dummy2", + "dummy2", + null, + 0, + ServerType.REALTIME, + "dummy", + 0 + ); + private static final DruidServer DUMMY_BROKER = new DruidServer( + "dummy3", + "dummy3", + null, + 0, + ServerType.BROKER, + "dummy", + 0 + ); + + private final Map> timelines; + private Map, Pair> segmentInfo; + private List segments = new ArrayList<>(); + private List realtimeSegments = new ArrayList<>(); + private List brokerSegments = new ArrayList<>(); + private List> timelineCallbackExecs = new ArrayList<>(); + + public TestCoordinatorServerView(List segments, List realtimeSegments) + { + super( + Mockito.mock(ServerInventoryView.class), + Mockito.mock(CoordinatorSegmentWatcherConfig.class), + Mockito.mock(ServiceEmitter.class), + Mockito.mock(DirectDruidClientFactory.class) + ); + + timelines = new HashMap<>(); + segmentInfo = new HashMap<>(); + + for (DataSegment segment : segments) { + addToTimeline(segment, DUMMY_SERVER); + } + + for (DataSegment realtimeSegment : realtimeSegments) { + addToTimeline(realtimeSegment, DUMMY_SERVER_REALTIME); + } + } + + private DruidServer getServerForType(ServerType serverType) + { + switch (serverType) { + case BROKER: + return DUMMY_BROKER; + case REALTIME: + return DUMMY_SERVER_REALTIME; + default: + return DUMMY_SERVER; + } + } + + private void addToTimeline(DataSegment dataSegment, DruidServer druidServer) + { + if (druidServer.getMetadata().getType() == ServerType.REALTIME) { + realtimeSegments.add(dataSegment); + } else if (druidServer.getMetadata().getType() == ServerType.BROKER) { + brokerSegments.add(dataSegment); + } else { + segments.add(dataSegment); + } + SegmentDescriptor segmentDescriptor = dataSegment.getId().toDescriptor(); + SegmentLoadInfo segmentLoadInfo = new SegmentLoadInfo(dataSegment); + segmentLoadInfo.addServer(druidServer.getMetadata()); + + segmentInfo.put(Pair.of(dataSegment, druidServer.getType()), Pair.of(druidServer.getMetadata(), segmentLoadInfo)); + + TableDataSource tableDataSource = new TableDataSource(dataSegment.getDataSource()); + timelines.computeIfAbsent(tableDataSource, value -> new VersionedIntervalTimeline<>(Comparator.naturalOrder())); + VersionedIntervalTimeline timeline = timelines.get(tableDataSource); + final ShardSpec shardSpec = new SingleDimensionShardSpec("dimAll", null, null, 0, 1); + timeline.add(dataSegment.getInterval(), segmentDescriptor.getVersion(), shardSpec.createChunk(segmentLoadInfo)); + } + + @Override + public QueryRunner getQueryRunner(String serverName) + { + return EasyMock.mock(QueryRunner.class); + } + + @Override + public VersionedIntervalTimeline getTimeline(DataSource dataSource) + { + return timelines.get(dataSource); + } + + @Override + public void registerTimelineCallback(final Executor exec, final TimelineServerView.TimelineCallback callback) + { + for (DataSegment segment : segments) { + exec.execute(() -> callback.segmentAdded(DUMMY_SERVER.getMetadata(), segment)); + } + for (DataSegment segment : realtimeSegments) { + exec.execute(() -> callback.segmentAdded(DUMMY_SERVER_REALTIME.getMetadata(), segment)); + } + exec.execute(callback::timelineInitialized); + timelineCallbackExecs.add(new Pair<>(exec, callback)); + } + + public void addSegment(DataSegment segment, ServerType serverType) + { + DruidServer druidServer = getServerForType(serverType); + addToTimeline(segment, druidServer); + + timelineCallbackExecs.forEach( + execAndCallback -> execAndCallback.lhs.execute(() -> execAndCallback.rhs.segmentAdded(druidServer.getMetadata(), segment)) + ); + } + + public void removeSegment(DataSegment segment, ServerType serverType) + { + DruidServerMetadata druidServerMetadata; + if (serverType == ServerType.BROKER) { + druidServerMetadata = DUMMY_BROKER.getMetadata(); + brokerSegments.remove(segment); + } else if (serverType == ServerType.REALTIME) { + druidServerMetadata = DUMMY_SERVER_REALTIME.getMetadata(); + realtimeSegments.remove(segment); + } else { + druidServerMetadata = DUMMY_SERVER.getMetadata(); + segments.remove(segment); + } + + Pair key = Pair.of(segment, serverType); + Pair info = segmentInfo.get(key); + + segmentInfo.remove(key); + + if (null != info) { + timelines.get(new TableDataSource(segment.getDataSource())).remove( + segment.getInterval(), + "0", + new SingleDimensionShardSpec("dimAll", null, null, 0, 1) + .createChunk(info.rhs) + ); + } + + timelineCallbackExecs.forEach( + execAndCallback -> execAndCallback.lhs.execute(() -> { + execAndCallback.rhs.serverSegmentRemoved(druidServerMetadata, segment); + + // Fire segmentRemoved if all replicas have been removed. + if (!segments.contains(segment) && !brokerSegments.contains(segment) && !realtimeSegments.remove(segment)) { + execAndCallback.rhs.segmentRemoved(segment); + } + }) + ); + } + + @Nullable + @Override + public List getInventory() + { + return Lists.newArrayList(DUMMY_SERVER, DUMMY_SERVER_REALTIME); + } + + public List getSegmentsOfServer(DruidServer druidServer) + { + if (druidServer.getType() == ServerType.BROKER) { + return Lists.newArrayList(brokerSegments); + } else if (druidServer.getType() == ServerType.REALTIME) { + return Lists.newArrayList(realtimeSegments); + } else { + return Lists.newArrayList(segments); + } + } +} diff --git a/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java b/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java new file mode 100644 index 000000000000..3db86cf64414 --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.guice.http.DruidHttpClientConfig; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.guava.FunctionalIterable; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.query.BySegmentQueryRunner; +import org.apache.druid.query.FinalizeResultsQueryRunner; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerFactory; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.SegmentDescriptor; +import org.apache.druid.query.context.ResponseContext; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.QueryableIndexSegment; +import org.apache.druid.server.initialization.ServerConfig; +import org.apache.druid.timeline.DataSegment; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class TestSegmentMetadataQueryWalker extends SegmentMetadataQuerySegmentWalker +{ + private QueryRunnerFactoryConglomerate conglomerate; + private Map> queryableIndexMap; + + public TestSegmentMetadataQueryWalker( + CoordinatorServerView serverView, + DruidHttpClientConfig httpClientConfig, + QueryToolChestWarehouse warehouse, + ServerConfig serverConfig, + ServiceEmitter emitter, + QueryRunnerFactoryConglomerate conglomerate, + Map> queryableIndexMap + ) + { + super( + serverView, + httpClientConfig, + warehouse, + serverConfig, + emitter + ); + this.conglomerate = conglomerate; + this.queryableIndexMap = queryableIndexMap; + } + + public void add(DataSegment segment, QueryableIndex index) + { + queryableIndexMap.put(segment.toDescriptor(), Pair.of(index, segment)); + } + + @Override + Sequence getServerResults( + QueryRunner serverRunner, + QueryPlus queryPlus, + ResponseContext responseContext, + long maxQueuedBytesPerServer, + List segmentDescriptors + ) + { + QueryRunnerFactory factory = conglomerate.findFactory(queryPlus.getQuery()); + QueryToolChest toolChest = factory.getToolchest(); + + return new FinalizeResultsQueryRunner<>( + toolChest.mergeResults( + factory.mergeRunners( + Execs.directExecutor(), + FunctionalIterable + .create(segmentDescriptors) + .transform( + segment -> + new BySegmentQueryRunner<>( + queryableIndexMap.get(segment).rhs.getId(), + queryableIndexMap.get(segment).rhs.getInterval().getStart(), + factory.createRunner( + new QueryableIndexSegment( + queryableIndexMap.get(segment).lhs, + queryableIndexMap.get(segment).rhs.getId())) + ) + ) + ) + ), + toolChest + ).run(queryPlus, responseContext); + } + + public List getSegments() + { + return queryableIndexMap.values() + .stream() + .map(pair -> pair.rhs) + .collect(Collectors.toList()); + } +} diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index e8f9f1e9fe92..8551a9859d76 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -400,7 +400,7 @@ public CallbackAction segmentViewInitialized() } }; - serverView = new CoordinatorServerView(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); + serverView = new CoordinatorServerView(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter(), null); baseView.start(); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 353b981c506e..32778a6d79bd 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -32,38 +32,31 @@ import com.google.inject.Provider; import com.google.inject.Provides; import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.MapBinder; import com.google.inject.name.Names; import com.google.inject.util.Providers; import org.apache.curator.framework.CuratorFramework; import org.apache.druid.audit.AuditManager; -import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; import org.apache.druid.client.CoordinatorServerView; -import org.apache.druid.client.CoordinatorTimeline; +import org.apache.druid.client.DirectDruidClientFactory; import org.apache.druid.client.HttpServerInventoryViewResource; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.client.QueryableCoordinatorServerView; -import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; import org.apache.druid.client.selector.ServerSelectorStrategy; import org.apache.druid.client.selector.TierSelectorStrategy; import org.apache.druid.discovery.NodeRole; -import org.apache.druid.guice.BrokerProcessingModule; -import org.apache.druid.guice.CacheModule; import org.apache.druid.guice.ConfigProvider; +import org.apache.druid.guice.DruidBinders; import org.apache.druid.guice.Jerseys; -import org.apache.druid.guice.JoinableFactoryModule; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; import org.apache.druid.guice.LazySingleton; -import org.apache.druid.guice.LegacyBrokerParallelMergeConfigModule; import org.apache.druid.guice.LifecycleModule; import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.guice.QueryRunnerFactoryModule; -import org.apache.druid.guice.QueryableModule; -import org.apache.druid.guice.SegmentWranglerModule; import org.apache.druid.guice.annotations.EscalatedGlobal; +import org.apache.druid.guice.annotations.Global; import org.apache.druid.guice.http.JettyHttpClientModule; import org.apache.druid.indexing.overlord.TaskMaster; import org.apache.druid.indexing.overlord.TaskStorage; @@ -84,13 +77,28 @@ import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.metadata.SegmentsMetadataManagerConfig; import org.apache.druid.metadata.SegmentsMetadataManagerProvider; +import org.apache.druid.query.DefaultGenericQueryMetricsFactory; +import org.apache.druid.query.DefaultQueryConfig; +import org.apache.druid.query.GenericQueryMetricsFactory; +import org.apache.druid.query.MapQueryToolChestWarehouse; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryRunnerFactory; import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; +import org.apache.druid.query.metadata.SegmentMetadataQueryConfig; +import org.apache.druid.query.metadata.SegmentMetadataQueryQueryToolChest; +import org.apache.druid.query.metadata.SegmentMetadataQueryRunnerFactory; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; -import org.apache.druid.server.ClientQuerySegmentWalker; +import org.apache.druid.segment.metadata.SegmentMetadataQuerySegmentWalker; +import org.apache.druid.server.QueryScheduler; +import org.apache.druid.server.QuerySchedulerProvider; import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.CoordinatorConfigManager; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -122,6 +130,9 @@ import org.apache.druid.server.http.TiersResource; import org.apache.druid.server.initialization.ZkPathsConfig; import org.apache.druid.server.initialization.jetty.JettyServerInitializer; +import org.apache.druid.server.log.NoopRequestLoggerProvider; +import org.apache.druid.server.log.RequestLogger; +import org.apache.druid.server.log.RequestLoggerProvider; import org.apache.druid.server.lookup.cache.LookupCoordinatorManager; import org.apache.druid.server.lookup.cache.LookupCoordinatorManagerConfig; import org.apache.druid.server.metrics.ServiceStatusMonitor; @@ -220,12 +231,13 @@ public void configure(Binder binder) binder.bind(RedirectInfo.class).to(CoordinatorRedirectInfo.class).in(LazySingleton.class); } + LifecycleModule.register(binder, CoordinatorServerView.class); + if (isSegmentMetadataCacheEnabled()) { - binder.install(new CoordinatorSegmentMetadataCacheModule(beOverlord)); + binder.install(new CoordinatorSegmentMetadataCacheModule()); } else { - binder.bind(CoordinatorTimeline.class).to(CoordinatorServerView.class).in(LazySingleton.class); - LifecycleModule.register(binder, CoordinatorServerView.class); binder.bind(CoordinatorSegmentMetadataCache.class).toProvider(Providers.of(null)); + binder.bind(DirectDruidClientFactory.class).toProvider(Providers.of(null)); } binder.bind(SegmentsMetadataManager.class) @@ -459,41 +471,54 @@ public Supplier> get() private static class CoordinatorSegmentMetadataCacheModule implements Module { - private final boolean beOverlord; - - public CoordinatorSegmentMetadataCacheModule(boolean beOverlord) - { - this.beOverlord = beOverlord; - } - @Override public void configure(Binder binder) { - // These modules are required to allow running queries on the Coordinator, - // since CoordinatorSegmentMetadataCache needs to query data nodes and tasks - binder.install(new LegacyBrokerParallelMergeConfigModule()); - binder.install(new QueryRunnerFactoryModule()); - binder.install(new SegmentWranglerModule()); - binder.install(new QueryableModule()); - binder.install(new BrokerProcessingModule()); - binder.install(new JoinableFactoryModule()); - if (!beOverlord) { - binder.install(new CacheModule()); - } - + MapBinder, QueryToolChest> toolChests = DruidBinders.queryToolChestBinder(binder); + + toolChests.addBinding(SegmentMetadataQuery.class).to(SegmentMetadataQueryQueryToolChest.class); + binder.bind(SegmentMetadataQueryQueryToolChest.class).in(LazySingleton.class); + binder.bind(QueryToolChestWarehouse.class).to(MapQueryToolChestWarehouse.class); + + final MapBinder, QueryRunnerFactory> queryFactoryBinder = DruidBinders.queryRunnerFactoryBinder( + binder + ); + queryFactoryBinder.addBinding(SegmentMetadataQuery.class).to(SegmentMetadataQueryRunnerFactory.class); + binder.bind(SegmentMetadataQueryRunnerFactory.class).in(LazySingleton.class); + + binder.bind(GenericQueryMetricsFactory.class).to(DefaultGenericQueryMetricsFactory.class); + + binder.bind(QueryScheduler.class) + .toProvider(Key.get(QuerySchedulerProvider.class, Global.class)) + .in(LazySingleton.class); + binder.bind(QuerySchedulerProvider.class).in(LazySingleton.class); + JsonConfigProvider.bind(binder, "druid.query.scheduler", QuerySchedulerProvider.class, Global.class); + + binder.bind(RequestLogger.class).toProvider(RequestLoggerProvider.class).in(ManageLifecycle.class); + JsonConfigProvider.bindWithDefault( + binder, + "druid.request.logging", + RequestLoggerProvider.class, + NoopRequestLoggerProvider.class + ); + + JsonConfigProvider.bind(binder, "druid.query.default", DefaultQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.query.segmentMetadata", SegmentMetadataQueryConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.query.select", TierSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.coordinator.query.select.tier.custom", CustomTierSelectorStrategyConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.query.balancer", ServerSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.coordinator.query.retryPolicy", RetryQueryRunnerConfig.class); - binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); - binder.bind(CachingClusteredClient.class).in(LazySingleton.class); - binder.bind(QueryableCoordinatorServerView.class).in(LazySingleton.class); - binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); - binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); - LifecycleModule.register(binder, QueryableCoordinatorServerView.class); + binder.bind(QuerySegmentWalker.class).to(SegmentMetadataQuerySegmentWalker.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorSegmentMetadataCache.class); } + + @LazySingleton + @Provides + public QueryWatcher getWatcher(QueryScheduler scheduler) + { + return scheduler; + } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index ecae501f591d..2d81822fd442 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -24,6 +24,7 @@ import com.google.common.collect.Sets; import com.google.inject.Inject; import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.ServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.CoordinatorClient; import org.apache.druid.common.guava.FutureUtils; @@ -34,8 +35,10 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; +import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import java.io.IOException; @@ -80,7 +83,6 @@ public BrokerSegmentMetadataCache( { super( queryLifecycleFactory, - serverView, config, escalator, internalQueryConfig, @@ -88,6 +90,52 @@ public BrokerSegmentMetadataCache( ); this.dataSourceMetadataFactory = dataSourceMetadataFactory; this.coordinatorClient = coordinatorClient; + + initServerViewTimelineCallback(serverView); + } + + private void initServerViewTimelineCallback(final TimelineServerView serverView) + { + serverView.registerTimelineCallback( + callbackExec, + new TimelineServerView.TimelineCallback() + { + @Override + public ServerView.CallbackAction timelineInitialized() + { + synchronized (lock) { + isServerViewInitialized = true; + lock.notifyAll(); + } + + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentAdded(final DruidServerMetadata server, final DataSegment segment) + { + addSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DataSegment segment) + { + removeSegment(segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction serverSegmentRemoved( + final DruidServerMetadata server, + final DataSegment segment + ) + { + removeServerSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + } + ); } /** @@ -159,7 +207,7 @@ private Map queryDataSourceInformation(Set polledDataSourceMetadata.put( dataSourceInformation.getDataSource(), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheCommon.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheCommon.java new file mode 100644 index 000000000000..e8c9baf98c04 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheCommon.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.segment.join.JoinConditionAnalysis; +import org.apache.druid.segment.join.Joinable; +import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.SegmentMetadataCacheCommon; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.sql.calcite.util.TestTimelineServerView; +import org.apache.druid.timeline.DataSegment; +import org.easymock.EasyMock; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +public class BrokerSegmentMetadataCacheCommon extends SegmentMetadataCacheCommon +{ + public List druidServers; + SegmentManager segmentManager; + Set segmentDataSourceNames; + Set joinableDataSourceNames; + JoinableFactory globalTableJoinable; + + public SpecificSegmentsQuerySegmentWalker walker; + public TestTimelineServerView serverView; + + public void setUp() throws Exception + { + setUpData(); + setUpCommon(); + + segmentDataSourceNames = Sets.newConcurrentHashSet(); + joinableDataSourceNames = Sets.newConcurrentHashSet(); + + segmentManager = new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + { + @Override + public Set getDataSourceNames() + { + return segmentDataSourceNames; + } + }; + + globalTableJoinable = new JoinableFactory() + { + @Override + public boolean isDirectlyJoinable(DataSource dataSource) + { + return dataSource instanceof GlobalTableDataSource && + joinableDataSourceNames.contains(((GlobalTableDataSource) dataSource).getName()); + } + + @Override + public Optional build( + DataSource dataSource, + JoinConditionAnalysis condition + ) + { + return Optional.empty(); + } + }; + + walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) + .add(segment1, index1) + .add(segment2, index2) + .add(segment3, index2) + .add(segment4, indexAuto1) + .add(segment5, indexAuto2); + + final List realtimeSegments = ImmutableList.of(realtimeSegment1); + serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); + druidServers = serverView.getDruidServers(); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java new file mode 100644 index 000000000000..954df595b5cd --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java @@ -0,0 +1,502 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.druid.client.BrokerSegmentWatcherConfig; +import org.apache.druid.client.BrokerServerView; +import org.apache.druid.client.DirectDruidClient; +import org.apache.druid.client.DirectDruidClientFactory; +import org.apache.druid.client.DruidServer; +import org.apache.druid.client.FilteredServerInventoryView; +import org.apache.druid.client.FilteringSegmentCallback; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.ServerView; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; +import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; +import org.apache.druid.client.selector.RandomServerSelectorStrategy; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.NonnullPair; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.join.MapJoinableFactory; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +public class BrokerSegmentMetadataCacheConcurrencyTest extends BrokerSegmentMetadataCacheCommon +{ + private static final String DATASOURCE = "datasource"; + static final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); + private File tmpDir; + private TestServerInventoryView inventoryView; + private BrokerServerView serverView; + private AbstractSegmentMetadataCache schema; + private ExecutorService exec; + + @Before + @Override + public void setUp() throws Exception + { + super.setUp(); + tmpDir = temporaryFolder.newFolder(); + // walker = new SpecificSegmentsQuerySegmentWalker(conglomerate); + inventoryView = new TestServerInventoryView(); + serverView = newBrokerServerView(inventoryView); + inventoryView.init(); + serverView.awaitInitialization(); + exec = Execs.multiThreaded(4, "DruidSchemaConcurrencyTest-%d"); + } + + @After + @Override + public void tearDown() throws Exception + { + super.tearDown(); + exec.shutdownNow(); + } + + /** + * This tests the contention between three components, {@link AbstractSegmentMetadataCache}, + * {@code InventoryView}, and {@link BrokerServerView}. It first triggers + * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} + * is overridden to sleep before doing real work. While refreshing + * {@code SegmentMetadataCache}, more new segments are added to + * {@code InventoryView}, which triggers updates of {@code BrokerServerView}. + * Finally, while {@code BrokerServerView} is updated, + * {@link BrokerServerView#getTimeline} is continuously called to mimic user query + * processing. All these calls must return without heavy contention. + */ + @Test(timeout = 30000L) + public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerViewGetTimeline() + throws InterruptedException, ExecutionException, TimeoutException + { + schema = new BrokerSegmentMetadataCache( + getQueryLifecycleFactory(walker), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataFactory(new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public RowSignature buildDruidTable(final String dataSource) + { + doInLock(() -> { + try { + // Mimic some heavy work done in lock in DruidSchema + Thread.sleep(5000); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + return super.buildDruidTable(dataSource); + } + }; + + int numExistingSegments = 100; + int numServers = 19; + CountDownLatch segmentLoadLatch = new CountDownLatch(numExistingSegments); + serverView.registerTimelineCallback( + Execs.directExecutor(), + new TimelineServerView.TimelineCallback() + { + @Override + public ServerView.CallbackAction timelineInitialized() + { + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + segmentLoadLatch.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(DataSegment segment) + { + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegment segment) + { + return ServerView.CallbackAction.CONTINUE; + } + } + ); + addSegmentsToCluster(0, numServers, numExistingSegments); + // Wait for all segments to be loaded in BrokerServerView + Assert.assertTrue(segmentLoadLatch.await(5, TimeUnit.SECONDS)); + + // Trigger refresh of DruidSchema. This will internally run the heavy work + // mimicked by the overridden buildDruidTable + Future refreshFuture = exec.submit(() -> { + schema.refresh( + walker.getSegments().stream().map(DataSegment::getId).collect(Collectors.toSet()), + Sets.newHashSet(DATASOURCE) + ); + return null; + }); + + // Trigger updates of BrokerServerView. This should be done asynchronously. + addSegmentsToCluster(numExistingSegments, numServers, 50); // add completely new segments + addReplicasToCluster(1, numServers, 30); // add replicas of the first 30 segments. + // for the first 30 segments, we will still have replicas. + // for the other 20 segments, they will be completely removed from the cluster. + removeSegmentsFromCluster(numServers, 50); + Assert.assertFalse(refreshFuture.isDone()); + + for (int i = 0; i < 1000; i++) { + boolean hasTimeline = exec.submit( + () -> serverView.getTimeline((new TableDataSource(DATASOURCE)).getAnalysis()) + .isPresent() + ).get(100, TimeUnit.MILLISECONDS); + Assert.assertTrue(hasTimeline); + // We want to call getTimeline while BrokerServerView is being updated. Sleep might help with timing. + Thread.sleep(2); + } + + refreshFuture.get(10, TimeUnit.SECONDS); + } + + /** + * This tests the contention between two methods of {@link AbstractSegmentMetadataCache}: + * {@link AbstractSegmentMetadataCache#refresh} and + * {@link AbstractSegmentMetadataCache#getSegmentMetadataSnapshot()}. It first triggers + * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} + * is overridden to sleep before doing real work. While refreshing + * {@code SegmentMetadataCache}, {@code getSegmentMetadataSnapshot()} is continuously + * called to mimic reading the segments table of SystemSchema. All these calls + * must return without heavy contention. + */ + @Test(timeout = 30000L) + public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() + throws InterruptedException, ExecutionException, TimeoutException + { + schema = new BrokerSegmentMetadataCache( + getQueryLifecycleFactory(walker), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataFactory(new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public RowSignature buildDruidTable(final String dataSource) + { + doInLock(() -> { + try { + // Mimic some heavy work done in lock in SegmentMetadataCache + Thread.sleep(5000); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + return super.buildDruidTable(dataSource); + } + }; + + int numExistingSegments = 100; + int numServers = 19; + CountDownLatch segmentLoadLatch = new CountDownLatch(numExistingSegments); + serverView.registerTimelineCallback( + Execs.directExecutor(), + new TimelineServerView.TimelineCallback() + { + @Override + public ServerView.CallbackAction timelineInitialized() + { + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + segmentLoadLatch.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(DataSegment segment) + { + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegment segment) + { + return ServerView.CallbackAction.CONTINUE; + } + } + ); + addSegmentsToCluster(0, numServers, numExistingSegments); + // Wait for all segments to be loaded in BrokerServerView + Assert.assertTrue(segmentLoadLatch.await(5, TimeUnit.SECONDS)); + + // Trigger refresh of SegmentMetadataCache. This will internally run the heavy work mimicked + // by the overridden buildDruidTable + Future refreshFuture = exec.submit(() -> { + schema.refresh( + walker.getSegments().stream().map(DataSegment::getId).collect(Collectors.toSet()), + Sets.newHashSet(DATASOURCE) + ); + return null; + }); + Assert.assertFalse(refreshFuture.isDone()); + + for (int i = 0; i < 1000; i++) { + Map segmentsMetadata = exec.submit( + () -> schema.getSegmentMetadataSnapshot() + ).get(100, TimeUnit.MILLISECONDS); + Assert.assertFalse(segmentsMetadata.isEmpty()); + // We want to call getTimeline while refreshing. Sleep might help with timing. + Thread.sleep(2); + } + + refreshFuture.get(10, TimeUnit.SECONDS); + } + + private void addSegmentsToCluster(int partitionIdStart, int numServers, int numSegments) + { + for (int i = 0; i < numSegments; i++) { + DataSegment segment = newSegment(i + partitionIdStart); + QueryableIndex index = newQueryableIndex(i + partitionIdStart); + //walker.add(segment, index); + int serverIndex = i % numServers; + inventoryView.addServerSegment(newServer("server_" + serverIndex), segment); + } + } + + private void addReplicasToCluster(int serverIndexOffFrom, int numServers, int numSegments) + { + for (int i = 0; i < numSegments; i++) { + DataSegment segment = newSegment(i); + int serverIndex = i % numServers + serverIndexOffFrom; + serverIndex = serverIndex < numServers ? serverIndex : serverIndex - numServers; + inventoryView.addServerSegment(newServer("server_" + serverIndex), segment); + } + } + + private void removeSegmentsFromCluster(int numServers, int numSegments) + { + for (int i = 0; i < numSegments; i++) { + DataSegment segment = newSegment(i); + int serverIndex = i % numServers; + inventoryView.removeServerSegment(newServer("server_" + serverIndex), segment); + } + } + + private static BrokerServerView newBrokerServerView(FilteredServerInventoryView baseView) + { + DirectDruidClientFactory druidClientFactory = EasyMock.createMock(DirectDruidClientFactory.class); + DirectDruidClient directDruidClient = EasyMock.mock(DirectDruidClient.class); + EasyMock.expect(druidClientFactory.makeDirectClient(EasyMock.anyObject(DruidServer.class))) + .andReturn(directDruidClient) + .anyTimes(); + + EasyMock.replay(druidClientFactory); + return new BrokerServerView( + druidClientFactory, + baseView, + new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), + new NoopServiceEmitter(), + new BrokerSegmentWatcherConfig() + ); + } + + private static DruidServer newServer(String name) + { + return new DruidServer( + name, + "host:8083", + "host:8283", + 1000L, + ServerType.HISTORICAL, + "tier", + 0 + ); + } + + private DataSegment newSegment(int partitionId) + { + return new DataSegment( + DATASOURCE, + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of(), + ImmutableList.of(), + new NumberedShardSpec(partitionId, 0), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + } + + private QueryableIndex newQueryableIndex(int partitionId) + { + return IndexBuilder.create() + .tmpDir(new File(tmpDir, "" + partitionId)) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1") + ) + .withRollup(false) + .build() + ) + .rows(ROWS1) + .buildMMappedIndex(); + } + + private static class TestServerInventoryView implements FilteredServerInventoryView + { + private final Map serverMap = new HashMap<>(); + private final Map> segmentsMap = new HashMap<>(); + private final List> segmentCallbacks = new ArrayList<>(); + private final List> serverRemovedCallbacks = new ArrayList<>(); + + private void init() + { + segmentCallbacks.forEach(pair -> pair.rhs.execute(pair.lhs::segmentViewInitialized)); + } + + private void addServerSegment(DruidServer server, DataSegment segment) + { + serverMap.put(server.getName(), server); + segmentsMap.computeIfAbsent(server.getName(), k -> new HashSet<>()).add(segment); + segmentCallbacks.forEach(pair -> pair.rhs.execute(() -> pair.lhs.segmentAdded(server.getMetadata(), segment))); + } + + private void removeServerSegment(DruidServer server, DataSegment segment) + { + segmentsMap.computeIfAbsent(server.getName(), k -> new HashSet<>()).remove(segment); + segmentCallbacks.forEach(pair -> pair.rhs.execute(() -> pair.lhs.segmentRemoved(server.getMetadata(), segment))); + } + + @SuppressWarnings("unused") + private void removeServer(DruidServer server) + { + serverMap.remove(server.getName()); + segmentsMap.remove(server.getName()); + serverRemovedCallbacks.forEach(pair -> pair.rhs.execute(() -> pair.lhs.serverRemoved(server))); + } + + @Override + public void registerSegmentCallback( + Executor exec, + ServerView.SegmentCallback callback, + Predicate> filter + ) + { + segmentCallbacks.add(new NonnullPair<>(new FilteringSegmentCallback(callback, filter), exec)); + } + + @Override + public void registerServerRemovedCallback(Executor exec, ServerView.ServerRemovedCallback callback) + { + serverRemovedCallbacks.add(new NonnullPair<>(callback, exec)); + } + + @Nullable + @Override + public DruidServer getInventoryValue(String serverKey) + { + return serverMap.get(serverKey); + } + + @Override + public Collection getInventory() + { + return serverMap.values(); + } + + @Override + public boolean isStarted() + { + return true; + } + + @Override + public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) + { + Set segments = segmentsMap.get(serverKey); + return segments != null && segments.contains(segment); + } + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index ad804b78e918..7436ecc07076 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -32,13 +32,15 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.coordinator.CoordinatorClient; import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.query.DataSource; +import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.TableDataSource; @@ -48,18 +50,12 @@ import org.apache.druid.segment.QueryableIndexStorageAdapter; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.RowSignature; -import org.apache.druid.segment.join.JoinConditionAnalysis; -import org.apache.druid.segment.join.Joinable; -import org.apache.druid.segment.join.JoinableFactory; -import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCacheCommon; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryResponse; -import org.apache.druid.server.SegmentManager; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -86,12 +82,12 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; -public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon +public class BrokerSegmentMetadataCacheTest extends BrokerSegmentMetadataCacheCommon { private static final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); // Timeout to allow (rapid) debugging, while not blocking tests with errors. @@ -101,46 +97,12 @@ public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); private CountDownLatch refreshLatch = new CountDownLatch(1); - SegmentManager segmentManager; - Set segmentDataSourceNames; - Set joinableDataSourceNames; - JoinableFactory globalTableJoinable; @Before + @Override public void setUp() throws Exception { - setUpCommon(); - setupData(); - segmentDataSourceNames = Sets.newConcurrentHashSet(); - joinableDataSourceNames = Sets.newConcurrentHashSet(); - - segmentManager = new SegmentManager(EasyMock.createMock(SegmentLoader.class)) - { - @Override - public Set getDataSourceNames() - { - return segmentDataSourceNames; - } - }; - - globalTableJoinable = new JoinableFactory() - { - @Override - public boolean isDirectlyJoinable(DataSource dataSource) - { - return dataSource instanceof GlobalTableDataSource && - joinableDataSourceNames.contains(((GlobalTableDataSource) dataSource).getName()); - } - - @Override - public Optional build( - DataSource dataSource, - JoinConditionAnalysis condition - ) - { - return Optional.empty(); - } - }; + super.setUp(); } @After @@ -504,21 +466,100 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt public void testAvailableSegmentMetadataNumRows() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkAvailableSegmentMetadataNumRows(schema); + + Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); + final List segments = segmentsMetadata.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // find the only segment with datasource "foo2" + final DataSegment existingSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(existingSegment); + final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); + // update AvailableSegmentMetadata of existingSegment with numRows=5 + AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); + schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); + // find a druidServer holding existingSegment + final Pair pair = druidServers + .stream() + .flatMap(druidServer -> druidServer + .iterateAllSegments() + .stream() + .filter(segment -> segment.getId().equals(existingSegment.getId())) + .map(segment -> Pair.of(druidServer, segment)) + ) + .findAny() + .orElse(null); + Assert.assertNotNull(pair); + final ImmutableDruidServer server = pair.lhs; + Assert.assertNotNull(server); + final DruidServerMetadata druidServerMetadata = server.getMetadata(); + // invoke SegmentMetadataCache#addSegment on existingSegment + schema.addSegment(druidServerMetadata, existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + // get the only segment with datasource "foo2" + final DataSegment currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); + Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); + // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before + Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); } @Test public void testNullDatasource() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkNullDatasource(schema); + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // segments contains two segments with datasource "foo" and one with datasource "foo2" + // let's remove the only segment with datasource "foo2" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkNullAvailableSegmentMetadata(schema); + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // remove one of the segments with datasource "foo" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } /** @@ -686,7 +727,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. - BrokerSegmentMetadataCache mySchema = new BrokerSegmentMetadataCache( + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( factoryMock, serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -697,7 +738,39 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception new NoopCoordinatorClient() ); - checkRunSegmentMetadataQueryWithContext(mySchema, factoryMock, lifecycleMock); + Map queryContext = ImmutableMap.of( + QueryContexts.PRIORITY_KEY, 5, + QueryContexts.BROKER_PARALLEL_MERGE_KEY, false + ); + + DataSegment segment = newSegment("test", 0); + List segmentIterable = ImmutableList.of(segment.getId()); + + // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. + SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(segment.getDataSource()), + new MultipleSpecificSegmentSpec( + segmentIterable.stream() + .map(SegmentId::toDescriptor).collect(Collectors.toList())), + new AllColumnIncluderator(), + false, + queryContext, + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); + // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context + EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) + .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); + + EasyMock.replay(factoryMock, lifecycleMock); + + schema.runSegmentMetadataQuery(segmentIterable); + + EasyMock.verify(factoryMock, lifecycleMock); } @Test @@ -745,6 +818,16 @@ public void removeSegment(final DataSegment segment) } }; - checkRefreshShouldEmitMetrics(schema, dataSource, emitter, addSegmentLatch); + List segments = ImmutableList.of( + newSegment(dataSource, 1), + newSegment(dataSource, 2) + ); + serverView.addSegment(segments.get(0), ServerType.HISTORICAL); + serverView.addSegment(segments.get(1), ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(dataSource)); + + emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); + emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index 754136d853d4..cebb9e5446f8 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -26,7 +26,6 @@ import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; -import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; @@ -34,6 +33,7 @@ import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; +import org.apache.druid.sql.calcite.util.TestTimelineServerView; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 46f6c5fda9b2..e9cb1430a544 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -79,7 +79,6 @@ import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; -import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.DruidNode; import org.apache.druid.server.QueryStackTests; @@ -101,6 +100,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.TestDataBuilder; +import org.apache.druid.sql.calcite.util.TestTimelineServerView; import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index 1624b27cdc83..97f8dac72200 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -49,7 +49,6 @@ import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.join.JoinableFactoryWrapper; -import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.server.DruidNode; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryScheduler; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 1f2460dc279a..c3d2dd9b2416 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -39,7 +39,6 @@ import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; -import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestTimelineServerView.java similarity index 99% rename from server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java rename to sql/src/test/java/org/apache/druid/sql/calcite/util/TestTimelineServerView.java index 42c219665909..dca45e92eb06 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestTimelineServerView.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.segment.metadata; +package org.apache.druid.sql.calcite.util; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; From 89d38456ba7c9b9a328c2dba482520b7e1616177 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 14 Oct 2023 18:40:56 +0530 Subject: [PATCH 55/82] fix build --- .../org/apache/druid/sql/calcite/CalciteArraysQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 086633d7e59c..cbd8b7eb9310 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -81,9 +81,9 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; From 270dbd59bb5c2d873ced421645b292c6b3e8bed0 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 14 Oct 2023 19:08:24 +0530 Subject: [PATCH 56/82] fix build --- .../benchmark/DruidSchemaInternRowSignatureBenchmark.java | 6 +++--- .../druid/client/coordinator/CoordinatorClientImpl.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 85a9931ab037..570861052e7a 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -22,8 +22,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.client.TimelineServerView; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; @@ -72,7 +72,7 @@ private static class SegmentMetadataCacheForBenchmark extends CoordinatorSegment { public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, - final TimelineServerView serverView, + final CoordinatorServerView serverView, final Escalator escalator, final InternalQueryConfig internalQueryConfig ) @@ -167,7 +167,7 @@ public void setup() { cache = new SegmentMetadataCacheForBenchmark( EasyMock.mock(QueryLifecycleFactory.class), - EasyMock.mock(TimelineServerView.class), + EasyMock.mock(CoordinatorServerView.class), null, null ); diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index c829502d88bb..7cd9906b16bd 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -36,8 +36,8 @@ import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; -import java.util.HashSet; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; From 2da23b8fd8c9f2f27077e093d194f6d62b4a6923 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 14 Oct 2023 22:02:19 +0530 Subject: [PATCH 57/82] Fix spelling, intellij-inspection, codeql bug --- docs/api-reference/legacy-metadata-api.md | 2 +- .../metadata/CoordinatorSegmentMetadataCacheCommon.java | 3 ++- .../metadata/SegmentMetadataQuerySegmentWalkerTest.java | 2 +- .../schema/BrokerSegmentMetadataCacheConcurrencyTest.java | 5 +++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/api-reference/legacy-metadata-api.md b/docs/api-reference/legacy-metadata-api.md index bef4152e485c..b573816fe23f 100644 --- a/docs/api-reference/legacy-metadata-api.md +++ b/docs/api-reference/legacy-metadata-api.md @@ -126,7 +126,7 @@ Returns a list of all segments for one or more specific datasources with the ful `GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments&datasources={dataSourceName1}&datasources={dataSourceName2}` -Additionally, returns the realtime segments for the speicified datasources, with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. +Additionally, returns the realtime segments for the specified datasources, with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. `GET /druid/coordinator/v1/metadata/datasources` diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheCommon.java index 8ce1e7fe2035..b1f63825eb04 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheCommon.java @@ -29,6 +29,7 @@ import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.timeline.DataSegment; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,7 +47,7 @@ public void setUp() throws Exception serverView = new TestCoordinatorServerView( Lists.newArrayList(segment1, segment2, segment3, segment4, segment5), - Lists.newArrayList(realtimeSegment1) + Collections.singletonList(realtimeSegment1) ); Map> queryableIndexMap = new HashMap<>(); diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java index d74a2deb6e80..7d5638b14d6a 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java @@ -92,7 +92,7 @@ public class SegmentMetadataQuerySegmentWalkerTest private Random random; @Before - public void setUp() throws Exception + public void setUp() { warehouse = new MapQueryToolChestWarehouse( ImmutableMap., QueryToolChest>builder() diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java index 954df595b5cd..3b189821e999 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java @@ -52,6 +52,7 @@ import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -99,7 +100,7 @@ public void setUp() throws Exception { super.setUp(); tmpDir = temporaryFolder.newFolder(); - // walker = new SpecificSegmentsQuerySegmentWalker(conglomerate); + walker = new SpecificSegmentsQuerySegmentWalker(conglomerate); inventoryView = new TestServerInventoryView(); serverView = newBrokerServerView(inventoryView); inventoryView.init(); @@ -333,7 +334,7 @@ private void addSegmentsToCluster(int partitionIdStart, int numServers, int numS for (int i = 0; i < numSegments; i++) { DataSegment segment = newSegment(i + partitionIdStart); QueryableIndex index = newQueryableIndex(i + partitionIdStart); - //walker.add(segment, index); + walker.add(segment, index); int serverIndex = i % numServers; inventoryView.addServerSegment(newServer("server_" + serverIndex), segment); } From 6f568a62a6ef25044e46086e131b006e1ebbfddb Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 14 Oct 2023 22:04:51 +0530 Subject: [PATCH 58/82] undo some changes in CachingClusteredClientTest --- .../org/apache/druid/client/CachingClusteredClientTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java index 6d3c33753d10..91cd39e54a59 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java @@ -267,12 +267,12 @@ public class CachingClusteredClientTest private Cache cache; private DruidServer[] servers; - public CachingClusteredClientTest() + public CachingClusteredClientTest(int randomSeed) { - this.random = new Random(10); + this.random = new Random(randomSeed); } - //@Parameterized.Parameters(name = "{0}") + @Parameterized.Parameters(name = "{0}") public static Iterable constructorFeeder() { return Lists.transform( From fe229c0737bee7ed2d3ac0cad04d8d35238b4136 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 14 Oct 2023 22:31:30 +0530 Subject: [PATCH 59/82] minor changes --- .../org/apache/druid/client/BrokerServerView.java | 14 ++++---------- .../metadata/TestSegmentMetadataQueryWalker.java | 5 +++-- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index 4579698a4201..af1100645dad 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -63,24 +63,18 @@ public class BrokerServerView implements TimelineServerView { private static final Logger log = new Logger(BrokerServerView.class); + private final Object lock = new Object(); private final ConcurrentMap clients = new ConcurrentHashMap<>(); + private final Map selectors = new HashMap<>(); + private final Map> timelines = new HashMap<>(); private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); - private final DirectDruidClientFactory druidClientFactory; private final TierSelectorStrategy tierSelectorStrategy; private final ServiceEmitter emitter; private final BrokerSegmentWatcherConfig segmentWatcherConfig; private final Predicate> segmentFilter; private final CountDownLatch initialized = new CountDownLatch(1); - protected final FilteredServerInventoryView baseView; - - protected final Object lock = new Object(); - - // Map of segmentIds and the set of server where the segment is present - protected final Map selectors = new HashMap<>(); - - // Map of datasource and segment timeline - protected final Map> timelines = new HashMap<>(); + private final FilteredServerInventoryView baseView; @Inject public BrokerServerView( diff --git a/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java b/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java index 3db86cf64414..d53162a6720f 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java @@ -27,6 +27,7 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.query.BySegmentQueryRunner; +import org.apache.druid.query.DirectQueryProcessingPool; import org.apache.druid.query.FinalizeResultsQueryRunner; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; @@ -77,7 +78,7 @@ public void add(DataSegment segment, QueryableIndex index) } @Override - Sequence getServerResults( + Sequence getServerResults( QueryRunner serverRunner, QueryPlus queryPlus, ResponseContext responseContext, @@ -91,7 +92,7 @@ Sequence getServerResults( return new FinalizeResultsQueryRunner<>( toolChest.mergeResults( factory.mergeRunners( - Execs.directExecutor(), + DirectQueryProcessingPool.INSTANCE, FunctionalIterable .create(segmentDescriptors) .transform( From 473b25ca5a190b7c15b0f7dedfd590a58bfb5120 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 14 Oct 2023 22:38:32 +0530 Subject: [PATCH 60/82] Fix typo in metric name --- .../druid/sql/calcite/schema/BrokerSegmentMetadataCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 2d81822fd442..31b2d7f9c325 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -207,7 +207,7 @@ private Map queryDataSourceInformation(Set polledDataSourceMetadata.put( dataSourceInformation.getDataSource(), From 39fb2484abde834498d924bbeb386c70c40eeb3b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 15 Oct 2023 09:09:26 +0530 Subject: [PATCH 61/82] temporarily enable feature on ITs --- .../cases/cluster/Common/environment-configs/coordinator.env | 4 +++- integration-tests/docker/environment-configs/coordinator | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env b/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env index 7a0b2ae3995f..ce3516f29c5a 100644 --- a/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env +++ b/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env @@ -33,7 +33,6 @@ druid_auth_basic_common_cacheDirectory=/tmp/authCache/coordinator druid_auth_unsecuredPaths=["/druid/coordinator/v1/loadqueue"] druid_server_https_crlPath=/tls/revocations.crl druid_coordinator_period_indexingPeriod=PT180000S - # 2x indexing period so that kill period is valid druid_coordinator_kill_period=PT360000S druid_coordinator_period=PT1S @@ -43,3 +42,6 @@ druid_coordinator_period=PT1S # long for the coordinator to notice changes. druid_manager_segments_pollDuration=PT5S druid_coordinator_period=PT10S + +# temp change to test table schema building on coordinator, to be reverted +druid_coordinator_centralizedSchemaManagement_enabled=true diff --git a/integration-tests/docker/environment-configs/coordinator b/integration-tests/docker/environment-configs/coordinator index c9226e1da69f..b07fc4f0a54d 100644 --- a/integration-tests/docker/environment-configs/coordinator +++ b/integration-tests/docker/environment-configs/coordinator @@ -43,3 +43,6 @@ druid_coordinator_period_indexingPeriod=PT180000S # 2x indexing period so that kill period is valid druid_coordinator_kill_period=PT360000S druid_coordinator_period=PT1S + +# temp change to test table schema building on coordinator, to be reverted +druid_coordinator_centralizedSchemaManagement_enabled=true From cac695abc9279d382022c29899fbc005f2c14a14 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 15 Oct 2023 09:22:31 +0530 Subject: [PATCH 62/82] fix checkstyle issue --- .../druid/segment/metadata/TestSegmentMetadataQueryWalker.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java b/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java index d53162a6720f..eb76ca30db96 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/TestSegmentMetadataQueryWalker.java @@ -22,7 +22,6 @@ import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.guice.http.DruidHttpClientConfig; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.guava.FunctionalIterable; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.emitter.service.ServiceEmitter; From eb3e3c1f0c6064aedddd5ddc92a8a37bd1420680 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 15 Oct 2023 16:21:00 +0530 Subject: [PATCH 63/82] Changes in CliCoordinator to conditionally add segment metadata cache related modules instead of installing them --- .../org/apache/druid/cli/CliCoordinator.java | 49 +++++++------------ 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 32778a6d79bd..01398c69ddab 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -43,9 +43,6 @@ import org.apache.druid.client.HttpServerInventoryViewResource; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; -import org.apache.druid.client.selector.ServerSelectorStrategy; -import org.apache.druid.client.selector.TierSelectorStrategy; import org.apache.druid.discovery.NodeRole; import org.apache.druid.guice.ConfigProvider; import org.apache.druid.guice.DruidBinders; @@ -55,6 +52,7 @@ import org.apache.druid.guice.LazySingleton; import org.apache.druid.guice.LifecycleModule; import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.guice.QueryableModule; import org.apache.druid.guice.annotations.EscalatedGlobal; import org.apache.druid.guice.annotations.Global; import org.apache.druid.guice.http.JettyHttpClientModule; @@ -130,9 +128,6 @@ import org.apache.druid.server.http.TiersResource; import org.apache.druid.server.initialization.ZkPathsConfig; import org.apache.druid.server.initialization.jetty.JettyServerInitializer; -import org.apache.druid.server.log.NoopRequestLoggerProvider; -import org.apache.druid.server.log.RequestLogger; -import org.apache.druid.server.log.RequestLoggerProvider; import org.apache.druid.server.lookup.cache.LookupCoordinatorManager; import org.apache.druid.server.lookup.cache.LookupCoordinatorManagerConfig; import org.apache.druid.server.metrics.ServiceStatusMonitor; @@ -164,6 +159,7 @@ public class CliCoordinator extends ServerRunnable private Properties properties; private boolean beOverlord; + private boolean isSegmentMetadataCacheEnabled; public CliCoordinator() { @@ -175,6 +171,7 @@ public void configure(Properties properties) { this.properties = properties; beOverlord = isOverlord(properties); + isSegmentMetadataCacheEnabled = isSegmentMetadataCacheEnabled(properties); if (beOverlord) { log.info("Coordinator is configured to act as Overlord as well (%s = true).", AS_OVERLORD_PROPERTY); @@ -196,6 +193,11 @@ protected List getModules() modules.add(JettyHttpClientModule.global()); + if (isSegmentMetadataCacheEnabled) { + modules.add(new CoordinatorSegmentMetadataCacheModule()); + modules.add(new QueryableModule()); + } + modules.add( new Module() { @@ -233,9 +235,7 @@ public void configure(Binder binder) LifecycleModule.register(binder, CoordinatorServerView.class); - if (isSegmentMetadataCacheEnabled()) { - binder.install(new CoordinatorSegmentMetadataCacheModule()); - } else { + if (!isSegmentMetadataCacheEnabled) { binder.bind(CoordinatorSegmentMetadataCache.class).toProvider(Providers.of(null)); binder.bind(DirectDruidClientFactory.class).toProvider(Providers.of(null)); } @@ -371,7 +371,7 @@ public static boolean isOverlord(Properties properties) return Boolean.parseBoolean(properties.getProperty(AS_OVERLORD_PROPERTY)); } - private boolean isSegmentMetadataCacheEnabled() + private boolean isSegmentMetadataCacheEnabled(Properties properties) { return Boolean.parseBoolean(properties.getProperty(CENTRALIZED_SCHEMA_MANAGEMENT_ENABLED)); } @@ -474,15 +474,19 @@ private static class CoordinatorSegmentMetadataCacheModule implements Module @Override public void configure(Binder binder) { - MapBinder, QueryToolChest> toolChests = DruidBinders.queryToolChestBinder(binder); + JsonConfigProvider.bind(binder, "druid.query.scheduler", QuerySchedulerProvider.class, Global.class); + JsonConfigProvider.bind(binder, "druid.query.default", DefaultQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.query.segmentMetadata", SegmentMetadataQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.retryPolicy", RetryQueryRunnerConfig.class); + MapBinder, QueryToolChest> toolChests = DruidBinders.queryToolChestBinder(binder); toolChests.addBinding(SegmentMetadataQuery.class).to(SegmentMetadataQueryQueryToolChest.class); binder.bind(SegmentMetadataQueryQueryToolChest.class).in(LazySingleton.class); binder.bind(QueryToolChestWarehouse.class).to(MapQueryToolChestWarehouse.class); - final MapBinder, QueryRunnerFactory> queryFactoryBinder = DruidBinders.queryRunnerFactoryBinder( - binder - ); + final MapBinder, QueryRunnerFactory> queryFactoryBinder = + DruidBinders.queryRunnerFactoryBinder(binder); queryFactoryBinder.addBinding(SegmentMetadataQuery.class).to(SegmentMetadataQueryRunnerFactory.class); binder.bind(SegmentMetadataQueryRunnerFactory.class).in(LazySingleton.class); @@ -492,23 +496,6 @@ public void configure(Binder binder) .toProvider(Key.get(QuerySchedulerProvider.class, Global.class)) .in(LazySingleton.class); binder.bind(QuerySchedulerProvider.class).in(LazySingleton.class); - JsonConfigProvider.bind(binder, "druid.query.scheduler", QuerySchedulerProvider.class, Global.class); - - binder.bind(RequestLogger.class).toProvider(RequestLoggerProvider.class).in(ManageLifecycle.class); - JsonConfigProvider.bindWithDefault( - binder, - "druid.request.logging", - RequestLoggerProvider.class, - NoopRequestLoggerProvider.class - ); - - JsonConfigProvider.bind(binder, "druid.query.default", DefaultQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.query.segmentMetadata", SegmentMetadataQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.coordinator.query.select", TierSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.coordinator.query.select.tier.custom", CustomTierSelectorStrategyConfig.class); - JsonConfigProvider.bind(binder, "druid.coordinator.query.balancer", ServerSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.coordinator.query.retryPolicy", RetryQueryRunnerConfig.class); binder.bind(QuerySegmentWalker.class).to(SegmentMetadataQuerySegmentWalker.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorSegmentMetadataCache.class); From 30438f40e2bb24cca9264801a2730a29f9797e69 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 16 Oct 2023 15:57:58 +0530 Subject: [PATCH 64/82] temporary changes to debug IT failure --- .github/workflows/revised-its.yml | 4 ++-- .github/workflows/standard-its.yml | 9 +++++---- .../workflows/unit-and-integration-tests-unified.yml | 11 ++++++----- .../cluster/Common/environment-configs/broker.env | 3 ++- .../Common/environment-configs/coordinator.env | 1 + integration-tests/docker/environment-configs/broker | 3 ++- .../docker/environment-configs/coordinator | 1 + .../calcite/schema/BrokerSegmentMetadataCache.java | 10 ++++++++-- .../schema/BrokerSegmentMetadataCacheConfig.java | 9 +++++++++ 9 files changed, 36 insertions(+), 15 deletions(-) diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index 900b308569d2..bb99cdf65033 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -50,7 +50,7 @@ jobs: matrix: #jdk: [8, 11, 17] jdk: [8] - it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat] + it: [MultiStageQuery, BatchIndex] #indexer: [indexer, middleManager] indexer: [middleManager] uses: ./.github/workflows/reusable-revised-its.yml @@ -66,7 +66,7 @@ jobs: s3-deep-storage-minio: needs: changes uses: ./.github/workflows/reusable-revised-its.yml - if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: false #${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 8 runtime_jdk: 11 diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index ae78a1f2a836..76c624e9df3f 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -49,7 +49,7 @@ jobs: matrix: testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction] uses: ./.github/workflows/reusable-standard-its.yml - if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: false #${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 8 runtime_jdk: 8 @@ -64,7 +64,7 @@ jobs: matrix: testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, append-ingestion, compaction] uses: ./.github/workflows/reusable-standard-its.yml - if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: false #${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 8 runtime_jdk: 8 @@ -112,7 +112,7 @@ jobs: matrix: indexer: [indexer, middleManager] uses: ./.github/workflows/reusable-standard-its.yml - if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: false #${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 8 runtime_jdk: 8 @@ -136,7 +136,7 @@ jobs: integration-k8s-leadership-tests: needs: changes name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test - if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: false # ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} runs-on: ubuntu-22.04 env: MVN: mvn --no-snapshot-updates @@ -184,6 +184,7 @@ jobs: done integration-other-tests: + if: false strategy: fail-fast: false matrix: diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 59ea3f95a585..2fbe5f1a9fa9 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -121,7 +121,7 @@ jobs: name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests - if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: false #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} with: jdk: ${{ matrix.jdk }} sql_compatibility: true @@ -134,16 +134,17 @@ jobs: name: "unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }})" uses: ./.github/workflows/unit-tests.yml needs: build + if: false with: jdk: 8 sql_compatibility: ${{ matrix.sql_compatibility }} standard-its: - needs: unit-tests - if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + needs: build + if: true #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/standard-its.yml revised-its: - needs: unit-tests - if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + needs: build + if: true #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/revised-its.yml diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/broker.env b/integration-tests-ex/cases/cluster/Common/environment-configs/broker.env index 652bbaf69df5..0ba78545d1d3 100644 --- a/integration-tests-ex/cases/cluster/Common/environment-configs/broker.env +++ b/integration-tests-ex/cases/cluster/Common/environment-configs/broker.env @@ -50,4 +50,5 @@ druid_query_scheduler_laning_strategy=manual druid_query_scheduler_laning_lanes_one=1 druid_segmentCache_locations=[{"path":"/shared/druid/brokerIndexCache","maxSize":1000000000}] druid_server_maxSize=1000000000 -druid_sql_planner_metadataRefreshPeriod=PT15S +druid_sql_planner_metadataRefreshPeriod=PT20S +druid_sql_planner_disableSegmentMetadataQueries=true diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env b/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env index ce3516f29c5a..3631b32d4396 100644 --- a/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env +++ b/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env @@ -45,3 +45,4 @@ druid_coordinator_period=PT10S # temp change to test table schema building on coordinator, to be reverted druid_coordinator_centralizedSchemaManagement_enabled=true +druid_query_segmentMetadata_metadataRefreshPeriod=PT15S \ No newline at end of file diff --git a/integration-tests/docker/environment-configs/broker b/integration-tests/docker/environment-configs/broker index b8d1849574b1..e8e282a74157 100644 --- a/integration-tests/docker/environment-configs/broker +++ b/integration-tests/docker/environment-configs/broker @@ -42,4 +42,5 @@ druid_query_scheduler_laning_strategy=manual druid_query_scheduler_laning_lanes_one=1 druid_segmentCache_locations=[{"path":"/shared/druid/brokerIndexCache","maxSize":1000000000}] druid_server_maxSize=1000000000 -druid_sql_planner_metadataRefreshPeriod=PT15S \ No newline at end of file +druid_sql_planner_metadataRefreshPeriod=PT20S +druid_sql_planner_disableSegmentMetadataQueries=true \ No newline at end of file diff --git a/integration-tests/docker/environment-configs/coordinator b/integration-tests/docker/environment-configs/coordinator index b07fc4f0a54d..14a99ef12a9b 100644 --- a/integration-tests/docker/environment-configs/coordinator +++ b/integration-tests/docker/environment-configs/coordinator @@ -46,3 +46,4 @@ druid_coordinator_period=PT1S # temp change to test table schema building on coordinator, to be reverted druid_coordinator_centralizedSchemaManagement_enabled=true +druid_query_segmentMetadata_metadataRefreshPeriod=PT15S diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 31b2d7f9c325..7a8b77ff812e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -69,6 +69,8 @@ public class BrokerSegmentMetadataCache extends AbstractSegmentMetadataCache segmentsToRefresh, final Set da // Remove segments of the datasource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); + Set refreshed = new HashSet<>(); + // Refresh the remaining segments. - final Set refreshed = refreshSegments(segmentsToRefresh); + if (!config.isDisableSegmentMetadataQueries()) { + refreshed = refreshSegments(segmentsToRefresh); + } synchronized (lock) { // Add missing segments back to the refresh list. diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 8aa70ac0c21b..b66939954157 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -50,6 +50,9 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig @JsonProperty private boolean awaitInitializationOnStart = true; + @JsonProperty + private boolean disableSegmentMetadataQueries = false; + public static BrokerSegmentMetadataCacheConfig create() { return new BrokerSegmentMetadataCacheConfig(); @@ -74,6 +77,11 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } + public boolean isDisableSegmentMetadataQueries() + { + return disableSegmentMetadataQueries; + } + /** * This property is overriden on the broker, so that the cache initialization blocks startup. */ @@ -90,6 +98,7 @@ public String toString() "metadataSegmentCacheEnable=" + metadataSegmentCacheEnable + ", metadataSegmentPollPeriod=" + metadataSegmentPollPeriod + ", awaitInitializationOnStart=" + awaitInitializationOnStart + + ", disableSegmentMetadataQueries=" + disableSegmentMetadataQueries + ", metadataRefreshPeriod=" + getMetadataRefreshPeriod() + ", metadataColumnTypeMergePolicy=" + getMetadataColumnTypeMergePolicy() + '}'; From e88ad002158cca932c7b8e45f803c89ea5b2db2d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 16 Oct 2023 17:32:54 +0530 Subject: [PATCH 65/82] revert temporary changes in gha --- .github/workflows/revised-its.yml | 4 ++-- .github/workflows/standard-its.yml | 9 ++++----- .../workflows/unit-and-integration-tests-unified.yml | 11 +++++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index bb99cdf65033..900b308569d2 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -50,7 +50,7 @@ jobs: matrix: #jdk: [8, 11, 17] jdk: [8] - it: [MultiStageQuery, BatchIndex] + it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat] #indexer: [indexer, middleManager] indexer: [middleManager] uses: ./.github/workflows/reusable-revised-its.yml @@ -66,7 +66,7 @@ jobs: s3-deep-storage-minio: needs: changes uses: ./.github/workflows/reusable-revised-its.yml - if: false #${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 8 runtime_jdk: 11 diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index 76c624e9df3f..ae78a1f2a836 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -49,7 +49,7 @@ jobs: matrix: testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction] uses: ./.github/workflows/reusable-standard-its.yml - if: false #${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 8 runtime_jdk: 8 @@ -64,7 +64,7 @@ jobs: matrix: testing_group: [input-source, perfect-rollup-parallel-batch-index, kafka-index, append-ingestion, compaction] uses: ./.github/workflows/reusable-standard-its.yml - if: false #${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 8 runtime_jdk: 8 @@ -112,7 +112,7 @@ jobs: matrix: indexer: [indexer, middleManager] uses: ./.github/workflows/reusable-standard-its.yml - if: false #${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: build_jdk: 8 runtime_jdk: 8 @@ -136,7 +136,7 @@ jobs: integration-k8s-leadership-tests: needs: changes name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test - if: false # ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} + if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} runs-on: ubuntu-22.04 env: MVN: mvn --no-snapshot-updates @@ -184,7 +184,6 @@ jobs: done integration-other-tests: - if: false strategy: fail-fast: false matrix: diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 2fbe5f1a9fa9..59ea3f95a585 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -121,7 +121,7 @@ jobs: name: "unit tests (jdk${{ matrix.jdk }}, sql-compat=true)" uses: ./.github/workflows/unit-tests.yml needs: unit-tests - if: false #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} with: jdk: ${{ matrix.jdk }} sql_compatibility: true @@ -134,17 +134,16 @@ jobs: name: "unit tests (jdk8, sql-compat=${{ matrix.sql_compatibility }})" uses: ./.github/workflows/unit-tests.yml needs: build - if: false with: jdk: 8 sql_compatibility: ${{ matrix.sql_compatibility }} standard-its: - needs: build - if: true #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + needs: unit-tests + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/standard-its.yml revised-its: - needs: build - if: true #${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} + needs: unit-tests + if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }} uses: ./.github/workflows/revised-its.yml From 61d130b6aed4c1dc422daafdf96ba8a8a7e8e855 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 16 Oct 2023 17:34:44 +0530 Subject: [PATCH 66/82] revert temporary changes to run ITs with this feature --- .../cases/cluster/Common/environment-configs/broker.env | 3 +-- .../cases/cluster/Common/environment-configs/coordinator.env | 5 +---- integration-tests/docker/environment-configs/broker | 3 +-- integration-tests/docker/environment-configs/coordinator | 4 ---- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/broker.env b/integration-tests-ex/cases/cluster/Common/environment-configs/broker.env index 0ba78545d1d3..652bbaf69df5 100644 --- a/integration-tests-ex/cases/cluster/Common/environment-configs/broker.env +++ b/integration-tests-ex/cases/cluster/Common/environment-configs/broker.env @@ -50,5 +50,4 @@ druid_query_scheduler_laning_strategy=manual druid_query_scheduler_laning_lanes_one=1 druid_segmentCache_locations=[{"path":"/shared/druid/brokerIndexCache","maxSize":1000000000}] druid_server_maxSize=1000000000 -druid_sql_planner_metadataRefreshPeriod=PT20S -druid_sql_planner_disableSegmentMetadataQueries=true +druid_sql_planner_metadataRefreshPeriod=PT15S diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env b/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env index 3631b32d4396..7a0b2ae3995f 100644 --- a/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env +++ b/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env @@ -33,6 +33,7 @@ druid_auth_basic_common_cacheDirectory=/tmp/authCache/coordinator druid_auth_unsecuredPaths=["/druid/coordinator/v1/loadqueue"] druid_server_https_crlPath=/tls/revocations.crl druid_coordinator_period_indexingPeriod=PT180000S + # 2x indexing period so that kill period is valid druid_coordinator_kill_period=PT360000S druid_coordinator_period=PT1S @@ -42,7 +43,3 @@ druid_coordinator_period=PT1S # long for the coordinator to notice changes. druid_manager_segments_pollDuration=PT5S druid_coordinator_period=PT10S - -# temp change to test table schema building on coordinator, to be reverted -druid_coordinator_centralizedSchemaManagement_enabled=true -druid_query_segmentMetadata_metadataRefreshPeriod=PT15S \ No newline at end of file diff --git a/integration-tests/docker/environment-configs/broker b/integration-tests/docker/environment-configs/broker index e8e282a74157..b8d1849574b1 100644 --- a/integration-tests/docker/environment-configs/broker +++ b/integration-tests/docker/environment-configs/broker @@ -42,5 +42,4 @@ druid_query_scheduler_laning_strategy=manual druid_query_scheduler_laning_lanes_one=1 druid_segmentCache_locations=[{"path":"/shared/druid/brokerIndexCache","maxSize":1000000000}] druid_server_maxSize=1000000000 -druid_sql_planner_metadataRefreshPeriod=PT20S -druid_sql_planner_disableSegmentMetadataQueries=true \ No newline at end of file +druid_sql_planner_metadataRefreshPeriod=PT15S \ No newline at end of file diff --git a/integration-tests/docker/environment-configs/coordinator b/integration-tests/docker/environment-configs/coordinator index 14a99ef12a9b..c9226e1da69f 100644 --- a/integration-tests/docker/environment-configs/coordinator +++ b/integration-tests/docker/environment-configs/coordinator @@ -43,7 +43,3 @@ druid_coordinator_period_indexingPeriod=PT180000S # 2x indexing period so that kill period is valid druid_coordinator_kill_period=PT360000S druid_coordinator_period=PT1S - -# temp change to test table schema building on coordinator, to be reverted -druid_coordinator_centralizedSchemaManagement_enabled=true -druid_query_segmentMetadata_metadataRefreshPeriod=PT15S From 2e4c45bada0b572884eb0936f8d3cad72dfcfefa Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 16 Oct 2023 18:58:56 +0530 Subject: [PATCH 67/82] update docs with the config for enabling feature --- docs/configuration/index.md | 3 ++- docs/operations/metrics.md | 8 ++++---- .../main/java/org/apache/druid/cli/CliCoordinator.java | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 4c0bb0c2b898..2b7c30c1eb80 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -867,6 +867,7 @@ These Coordinator static configurations can be defined in the `coordinator/runti |`druid.coordinator.loadqueuepeon.repeatDelay`|The start and repeat delay for the loadqueuepeon, which manages the load and drop of segments.|PT0.050S (50 ms)| |`druid.coordinator.asOverlord.enabled`|Boolean value for whether this Coordinator process should act like an Overlord as well. This configuration allows users to simplify a druid cluster by not having to deploy any standalone Overlord processes. If set to true, then Overlord console is available at `http://coordinator-host:port/console.html` and be sure to set `druid.coordinator.asOverlord.overlordService` also. See next.|false| |`druid.coordinator.asOverlord.overlordService`| Required, if `druid.coordinator.asOverlord.enabled` is `true`. This must be same value as `druid.service` on standalone Overlord processes and `druid.selectors.indexing.serviceName` on Middle Managers.|NULL| +|`druid.coordinator.centralizedSchemaManagement.enabled`|Boolean flag for enabling table schema building on the Coordinator.|false| ##### Metadata Management @@ -2002,7 +2003,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.useApproximateTopN`|Whether to use approximate [TopN queries](../querying/topnquery.md) when a SQL query could be expressed as such. If false, exact [GroupBy queries](../querying/groupbyquery.md) will be used instead.|true| |`druid.sql.planner.requireTimeCondition`|Whether to require SQL to have filter conditions on __time column so that all generated native queries will have user specified intervals. If true, all queries without filter condition on __time column will fail|false| |`druid.sql.planner.sqlTimeZone`|Sets the default time zone for the server, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|UTC| -|`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false| +|`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|true| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| |`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|true| diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md index dc5011752e76..df8ce218bde2 100644 --- a/docs/operations/metrics.md +++ b/docs/operations/metrics.md @@ -72,9 +72,9 @@ Most metric values reset each emission period, as specified in `druid.monitoring |`metadatacache/init/time`|Time taken to initialize the broker segment metadata cache. Useful to detect if brokers are taking too long to start||Depends on the number of segments.| |`metadatacache/refresh/count`|Number of segments to refresh in broker segment metadata cache.|`dataSource`| |`metadatacache/refresh/time`|Time taken to refresh segments in broker segment metadata cache.|`dataSource`| -|`metadatacache/schemaPoll/count`|Number of coordinator polls to fetch datasource schema.|`dataSource`| -|`metadatacache/schemaPoll/failed`|Number of failed coordinator polls to fetch datasource schema.|`dataSource`| -|`metadatacache/schemaPoll/time`|Time taken for coordinator polls to fetch datasource schema.|`dataSource`| +|`metadatacache/schemaPoll/count`|Number of coordinator polls to fetch datasource schema.|| +|`metadatacache/schemaPoll/failed`|Number of failed coordinator polls to fetch datasource schema.|| +|`metadatacache/schemaPoll/time`|Time taken for coordinator polls to fetch datasource schema.|| |`serverview/sync/healthy`|Sync status of the Broker with a segment-loading server such as a Historical or Peon. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled. This metric can be used in conjunction with `serverview/sync/unstableTime` to debug slow startup of Brokers.|`server`, `tier`|1 for fully synced servers, 0 otherwise| |`serverview/sync/unstableTime`|Time in milliseconds for which the Broker has been failing to sync with a segment-loading server. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled.|`server`, `tier`|Not emitted for synced servers.| |`subquery/rowLimit/count`|Number of subqueries whose results are materialized as rows (Java objects on heap).|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | @@ -361,7 +361,7 @@ These metrics are for the Druid Coordinator and are reset each time the Coordina |`serverview/init/time`|Time taken to initialize the coordinator server view.||Depends on the number of segments.| |`serverview/sync/healthy`|Sync status of the Coordinator with a segment-loading server such as a Historical or Peon. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled. You can use this metric in conjunction with `serverview/sync/unstableTime` to debug slow startup of the Coordinator.|`server`, `tier`|1 for fully synced servers, 0 otherwise| |`serverview/sync/unstableTime`|Time in milliseconds for which the Coordinator has been failing to sync with a segment-loading server. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled.|`server`, `tier`|Not emitted for synced servers.| -|`metadatacache/init/time`|Time taken to initialize the coordinator segment metadata cache.|`dataSource`|Depends on the number of segments.| +|`metadatacache/init/time`|Time taken to initialize the coordinator segment metadata cache.||Depends on the number of segments.| |`metadatacache/refresh/count`|Number of segments to refresh in coordinator segment metadata cache.|`dataSource`| |`metadatacache/refresh/time`|Time taken to refresh segments in coordinator segment metadata cache.|`dataSource`| diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 01398c69ddab..d236bcb3dfcc 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -474,9 +474,9 @@ private static class CoordinatorSegmentMetadataCacheModule implements Module @Override public void configure(Binder binder) { - JsonConfigProvider.bind(binder, "druid.query.scheduler", QuerySchedulerProvider.class, Global.class); - JsonConfigProvider.bind(binder, "druid.query.default", DefaultQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.query.segmentMetadata", SegmentMetadataQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.scheduler", QuerySchedulerProvider.class, Global.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.default", DefaultQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.segmentMetadata", SegmentMetadataQueryConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.query.retryPolicy", RetryQueryRunnerConfig.class); From 255cf2c67a8690bd5851b4da3cc689fa9e3bad12 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 16 Oct 2023 18:58:56 +0530 Subject: [PATCH 68/82] update docs with the config for enabling feature --- docs/configuration/index.md | 3 ++- docs/operations/metrics.md | 8 ++++---- .../main/java/org/apache/druid/cli/CliCoordinator.java | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 4c0bb0c2b898..8c7122bedd53 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -867,6 +867,7 @@ These Coordinator static configurations can be defined in the `coordinator/runti |`druid.coordinator.loadqueuepeon.repeatDelay`|The start and repeat delay for the loadqueuepeon, which manages the load and drop of segments.|PT0.050S (50 ms)| |`druid.coordinator.asOverlord.enabled`|Boolean value for whether this Coordinator process should act like an Overlord as well. This configuration allows users to simplify a druid cluster by not having to deploy any standalone Overlord processes. If set to true, then Overlord console is available at `http://coordinator-host:port/console.html` and be sure to set `druid.coordinator.asOverlord.overlordService` also. See next.|false| |`druid.coordinator.asOverlord.overlordService`| Required, if `druid.coordinator.asOverlord.enabled` is `true`. This must be same value as `druid.service` on standalone Overlord processes and `druid.selectors.indexing.serviceName` on Middle Managers.|NULL| +|`druid.coordinator.centralizedTableSchema.enabled`|Boolean flag for enabling table schema building on the Coordinator.|false| ##### Metadata Management @@ -2002,7 +2003,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.useApproximateTopN`|Whether to use approximate [TopN queries](../querying/topnquery.md) when a SQL query could be expressed as such. If false, exact [GroupBy queries](../querying/groupbyquery.md) will be used instead.|true| |`druid.sql.planner.requireTimeCondition`|Whether to require SQL to have filter conditions on __time column so that all generated native queries will have user specified intervals. If true, all queries without filter condition on __time column will fail|false| |`druid.sql.planner.sqlTimeZone`|Sets the default time zone for the server, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|UTC| -|`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false| +|`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|true| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| |`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|true| diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md index dc5011752e76..df8ce218bde2 100644 --- a/docs/operations/metrics.md +++ b/docs/operations/metrics.md @@ -72,9 +72,9 @@ Most metric values reset each emission period, as specified in `druid.monitoring |`metadatacache/init/time`|Time taken to initialize the broker segment metadata cache. Useful to detect if brokers are taking too long to start||Depends on the number of segments.| |`metadatacache/refresh/count`|Number of segments to refresh in broker segment metadata cache.|`dataSource`| |`metadatacache/refresh/time`|Time taken to refresh segments in broker segment metadata cache.|`dataSource`| -|`metadatacache/schemaPoll/count`|Number of coordinator polls to fetch datasource schema.|`dataSource`| -|`metadatacache/schemaPoll/failed`|Number of failed coordinator polls to fetch datasource schema.|`dataSource`| -|`metadatacache/schemaPoll/time`|Time taken for coordinator polls to fetch datasource schema.|`dataSource`| +|`metadatacache/schemaPoll/count`|Number of coordinator polls to fetch datasource schema.|| +|`metadatacache/schemaPoll/failed`|Number of failed coordinator polls to fetch datasource schema.|| +|`metadatacache/schemaPoll/time`|Time taken for coordinator polls to fetch datasource schema.|| |`serverview/sync/healthy`|Sync status of the Broker with a segment-loading server such as a Historical or Peon. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled. This metric can be used in conjunction with `serverview/sync/unstableTime` to debug slow startup of Brokers.|`server`, `tier`|1 for fully synced servers, 0 otherwise| |`serverview/sync/unstableTime`|Time in milliseconds for which the Broker has been failing to sync with a segment-loading server. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled.|`server`, `tier`|Not emitted for synced servers.| |`subquery/rowLimit/count`|Number of subqueries whose results are materialized as rows (Java objects on heap).|This metric is only available if the `SubqueryCountStatsMonitor` module is included.| | @@ -361,7 +361,7 @@ These metrics are for the Druid Coordinator and are reset each time the Coordina |`serverview/init/time`|Time taken to initialize the coordinator server view.||Depends on the number of segments.| |`serverview/sync/healthy`|Sync status of the Coordinator with a segment-loading server such as a Historical or Peon. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled. You can use this metric in conjunction with `serverview/sync/unstableTime` to debug slow startup of the Coordinator.|`server`, `tier`|1 for fully synced servers, 0 otherwise| |`serverview/sync/unstableTime`|Time in milliseconds for which the Coordinator has been failing to sync with a segment-loading server. Emitted only when [HTTP-based server view](../configuration/index.md#segment-management) is enabled.|`server`, `tier`|Not emitted for synced servers.| -|`metadatacache/init/time`|Time taken to initialize the coordinator segment metadata cache.|`dataSource`|Depends on the number of segments.| +|`metadatacache/init/time`|Time taken to initialize the coordinator segment metadata cache.||Depends on the number of segments.| |`metadatacache/refresh/count`|Number of segments to refresh in coordinator segment metadata cache.|`dataSource`| |`metadatacache/refresh/time`|Time taken to refresh segments in coordinator segment metadata cache.|`dataSource`| diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 01398c69ddab..db7c3bfdfd48 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -155,7 +155,7 @@ public class CliCoordinator extends ServerRunnable { private static final Logger log = new Logger(CliCoordinator.class); private static final String AS_OVERLORD_PROPERTY = "druid.coordinator.asOverlord.enabled"; - private static final String CENTRALIZED_SCHEMA_MANAGEMENT_ENABLED = "druid.coordinator.centralizedSchemaManagement.enabled"; + private static final String CENTRALIZED_SCHEMA_MANAGEMENT_ENABLED = "druid.coordinator.centralizedTableSchema.enabled"; private Properties properties; private boolean beOverlord; @@ -474,11 +474,11 @@ private static class CoordinatorSegmentMetadataCacheModule implements Module @Override public void configure(Binder binder) { - JsonConfigProvider.bind(binder, "druid.query.scheduler", QuerySchedulerProvider.class, Global.class); - JsonConfigProvider.bind(binder, "druid.query.default", DefaultQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.query.segmentMetadata", SegmentMetadataQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.segmentMetadata", SegmentMetadataQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.scheduler", QuerySchedulerProvider.class, Global.class); + JsonConfigProvider.bind(binder, "druid.coordinator.query.default", DefaultQueryConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.query.retryPolicy", RetryQueryRunnerConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); MapBinder, QueryToolChest> toolChests = DruidBinders.queryToolChestBinder(binder); toolChests.addBinding(SegmentMetadataQuery.class).to(SegmentMetadataQueryQueryToolChest.class); From 1a6dfc5783f8b2c6ba7a80f03215f76678c5ae0b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 17 Oct 2023 13:56:19 +0530 Subject: [PATCH 69/82] Add IT for the feature --- .github/workflows/standard-its.yml | 4 +- ...ocker-compose.centralized-table-schema.yml | 105 ++++++++++++++++++ integration-tests/docker/druid.sh | 2 +- .../centralized-table-schema-sample-data.sql | 20 ++++ .../script/docker_compose_args.sh | 6 +- .../org/apache/druid/tests/TestNGGroup.java | 2 + .../tests/query/ITBroadcastJoinQueryTest.java | 2 +- .../druid/tests/query/ITJdbcQueryTest.java | 2 +- .../druid/tests/query/ITSqlCancelTest.java | 2 +- .../tests/query/ITSystemTableQueryTest.java | 2 +- .../druid/tests/query/ITTwitterQueryTest.java | 2 +- .../druid/tests/query/ITUnionQueryTest.java | 2 +- .../tests/query/ITWikipediaQueryTest.java | 2 +- 13 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 integration-tests/docker/docker-compose.centralized-table-schema.yml create mode 100644 integration-tests/docker/test-data/centralized-table-schema-sample-data.sql diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index ae78a1f2a836..29435cf9c346 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -77,7 +77,7 @@ jobs: strategy: fail-fast: false matrix: - testing_group: [query, query-retry, query-error, security, high-availability] + testing_group: [query, query-retry, query-error, security, high-availability, centralized-table-schema] uses: ./.github/workflows/reusable-standard-its.yml if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} with: @@ -192,6 +192,6 @@ jobs: with: build_jdk: 8 runtime_jdk: 8 - testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties + testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-table-schema use_indexer: ${{ matrix.indexer }} group: other diff --git a/integration-tests/docker/docker-compose.centralized-table-schema.yml b/integration-tests/docker/docker-compose.centralized-table-schema.yml new file mode 100644 index 000000000000..58f0f622c48a --- /dev/null +++ b/integration-tests/docker/docker-compose.centralized-table-schema.yml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2.2" +services: + druid-zookeeper-kafka: + extends: + file: docker-compose.base.yml + service: druid-zookeeper-kafka + + druid-metadata-storage: + extends: + file: docker-compose.base.yml + service: druid-metadata-storage + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + + druid-coordinator: + extends: + file: docker-compose.base.yml + service: druid-coordinator + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + - druid_coordinator_centralizedTableSchema_enabled=true + - druid_coordinator_segmentMetadata_metadataRefreshPeriod=PT15S + depends_on: + - druid-overlord + - druid-metadata-storage + - druid-zookeeper-kafka + + druid-overlord: + extends: + file: docker-compose.base.yml + service: druid-overlord + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-metadata-storage + - druid-zookeeper-kafka + + druid-historical: + extends: + file: docker-compose.base.yml + service: druid-historical + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + + druid-middlemanager: + extends: + file: docker-compose.base.yml + service: druid-middlemanager + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + - druid-overlord + + druid-broker: + extends: + file: docker-compose.base.yml + service: druid-broker + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + - druid_sql_planner_metadataRefreshPeriod=PT20S + - druid_sql_planner_disableSegmentMetadataQueries=true + depends_on: + - druid-coordinator + - druid-zookeeper-kafka + - druid-middlemanager + - druid-historical + + druid-router: + extends: + file: docker-compose.base.yml + service: druid-router + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + - druid-coordinator + - druid-broker + - druid-overlord + +networks: + druid-it-net: + name: druid-it-net + ipam: + config: + - subnet: 172.172.172.0/24 diff --git a/integration-tests/docker/druid.sh b/integration-tests/docker/druid.sh index eb13fe8b5271..f009197ed494 100755 --- a/integration-tests/docker/druid.sh +++ b/integration-tests/docker/druid.sh @@ -85,7 +85,7 @@ setupData() # The "query" and "security" test groups require data to be setup before running the tests. # In particular, they requires segments to be download from a pre-existing s3 bucket. # This is done by using the loadSpec put into metadatastore and s3 credientials set below. - if [ "$DRUID_INTEGRATION_TEST_GROUP" = "query" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-retry" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-error" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "high-availability" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "ldap-security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "upgrade" ]; then + if [ "$DRUID_INTEGRATION_TEST_GROUP" = "query" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-retry" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-error" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "high-availability" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "ldap-security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "upgrade" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-table-schema" ]; then # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72. find /var/lib/mysql -type f -exec touch {} \; && service mysql start \ && cat /test-data/${DRUID_INTEGRATION_TEST_GROUP}-sample-data.sql | mysql -u root druid \ diff --git a/integration-tests/docker/test-data/centralized-table-schema-sample-data.sql b/integration-tests/docker/test-data/centralized-table-schema-sample-data.sql new file mode 100644 index 000000000000..abe0f115189b --- /dev/null +++ b/integration-tests/docker/test-data/centralized-table-schema-sample-data.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9','twitterstream','2013-05-13T01:08:18.192Z','2013-01-01T00:00:00.000Z','2013-01-02T00:00:00.000Z',0,'2013-01-02T04:13:41.980Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-01T00:00:00.000Z/2013-01-02T00:00:00.000Z\",\"version\":\"2013-01-02T04:13:41.980Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/2013-01-02T04:13:41.980Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":445235220,\"identifier\":\"twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9\"}','1970-01-01T00:00:00.000Z'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9','twitterstream','2013-05-13T00:03:28.640Z','2013-01-02T00:00:00.000Z','2013-01-03T00:00:00.000Z',0,'2013-01-03T03:44:58.791Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-02T00:00:00.000Z/2013-01-03T00:00:00.000Z\",\"version\":\"2013-01-03T03:44:58.791Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z/2013-01-03T03:44:58.791Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":435325540,\"identifier\":\"twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9\"}','1970-01-01T00:00:00.000Z'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9','twitterstream','2013-05-13T00:03:48.807Z','2013-01-03T00:00:00.000Z','2013-01-04T00:00:00.000Z',0,'2013-01-04T04:09:13.590Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-03T00:00:00.000Z/2013-01-04T00:00:00.000Z\",\"version\":\"2013-01-04T04:09:13.590Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z/2013-01-04T04:09:13.590Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":411651320,\"identifier\":\"twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9\"}','1970-01-01T00:00:00.000Z'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload,used_status_last_updated) VALUES ('wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9','wikipedia_editstream','2013-03-15T20:49:52.348Z','2012-12-29T00:00:00.000Z','2013-01-10T08:00:00.000Z',0,'2013-01-10T08:13:47.830Z_v9',1,'{\"dataSource\":\"wikipedia_editstream\",\"interval\":\"2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z\",\"version\":\"2013-01-10T08:13:47.830Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia_editstream/2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z/2013-01-10T08:13:47.830Z_v9/0/index.zip\"},\"dimensions\":\"anonymous,area_code,city,continent_code,country_name,dma_code,geo,language,namespace,network,newpage,page,postal_code,region_lookup,robot,unpatrolled,user\",\"metrics\":\"added,count,deleted,delta,delta_hist,unique_users,variation\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":446027801,\"identifier\":\"wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9\"}','1970-01-01T00:00:00.000Z'); +INSERT INTO druid_segments (id, dataSource, created_date, start, end, partitioned, version, used, payload,used_status_last_updated) VALUES ('wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', 'wikipedia', '2013-08-08T21:26:23.799Z', '2013-08-01T00:00:00.000Z', '2013-08-02T00:00:00.000Z', '0', '2013-08-08T21:22:48.989Z', '1', '{\"dataSource\":\"wikipedia\",\"interval\":\"2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z\",\"version\":\"2013-08-08T21:22:48.989Z\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip\"},\"dimensions\":\"dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace,anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup\",\"metrics\":\"count,delta,variation,added,deleted\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":24664730,\"identifier\":\"wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z\"}','1970-01-01T00:00:00.000Z'); diff --git a/integration-tests/script/docker_compose_args.sh b/integration-tests/script/docker_compose_args.sh index ac0f9533c159..f2f98cd6c03c 100644 --- a/integration-tests/script/docker_compose_args.sh +++ b/integration-tests/script/docker_compose_args.sh @@ -16,7 +16,7 @@ set -e -# picks appropriate docker-compose argments to use when bringing up and down integration test clusters +# picks appropriate docker-compose arguments to use when bringing up and down integration test clusters # for a given test group getComposeArgs() { @@ -71,6 +71,10 @@ getComposeArgs() then # default + with override config + schema registry container echo "-f ${DOCKERDIR}/docker-compose.yml -f ${DOCKERDIR}/docker-compose.schema-registry.yml" + elif [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-table-schema" ] + then + # cluster with overriden properties for broker and coordinator + echo "-f ${DOCKERDIR}/docker-compose.centralized-table-schema.yml" else # default echo "-f ${DOCKERDIR}/docker-compose.yml" diff --git a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java index aeb8a010efd8..e3703ec71c9d 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java @@ -161,4 +161,6 @@ public class TestNGGroup public static final String CUSTOM_COORDINATOR_DUTIES = "custom-coordinator-duties"; public static final String HTTP_ENDPOINT = "http-endpoint"; + + public static final String CENTRALIZED_TABLE_SCHEMA = "centralized-table-schema"; } diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java index 22323edef746..09c2b6c49bcc 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java @@ -39,7 +39,7 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; -@Test(groups = TestNGGroup.QUERY) +@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA}) @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITBroadcastJoinQueryTest extends AbstractIndexerTest { diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java index dbec477559d2..e942fc6c6296 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java @@ -47,7 +47,7 @@ import java.util.Properties; import java.util.Set; -@Test(groups = TestNGGroup.QUERY) +@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA}) @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITJdbcQueryTest { diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java index d5bc1e30204f..dcbae71c37f9 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java @@ -44,7 +44,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -@Test(groups = TestNGGroup.QUERY) +@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA}) @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITSqlCancelTest { diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java index df00488c6afa..9bcb47f1912d 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java @@ -29,7 +29,7 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; -@Test(groups = TestNGGroup.QUERY) +@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA}) @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITSystemTableQueryTest { diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITTwitterQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITTwitterQueryTest.java index f0ecaa011d9a..41486902add3 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITTwitterQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITTwitterQueryTest.java @@ -29,7 +29,7 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; -@Test(groups = TestNGGroup.QUERY) +@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA}) @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITTwitterQueryTest { diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITUnionQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITUnionQueryTest.java index 118218322af5..2e826609d729 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITUnionQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITUnionQueryTest.java @@ -58,7 +58,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; -@Test(groups = TestNGGroup.QUERY) +@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA}) @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITUnionQueryTest extends AbstractIndexerTest { diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITWikipediaQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITWikipediaQueryTest.java index ee53ab5f7873..ccbf73e4776e 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITWikipediaQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITWikipediaQueryTest.java @@ -47,7 +47,7 @@ import java.util.UUID; import java.util.concurrent.Future; -@Test(groups = TestNGGroup.QUERY) +@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA}) @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITWikipediaQueryTest { From 4b51c428ba532d49da1e018d8d83f29b8b92a0e5 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 20 Oct 2023 23:47:14 +0530 Subject: [PATCH 70/82] Changes in BrokerSegmentMetadataCache to poll schema for all the local datasources in each refresh cycle --- .../schema/BrokerSegmentMetadataCache.java | 10 ++-- .../BrokerSegmentMetadataCacheTest.java | 47 +++++++++++++++++-- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 7a8b77ff812e..825ad879078d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -19,7 +19,6 @@ package org.apache.druid.sql.calcite.schema; -import com.google.common.base.Predicates; import com.google.common.base.Stopwatch; import com.google.common.collect.Sets; import com.google.inject.Inject; @@ -155,16 +154,19 @@ public ServerView.CallbackAction serverSegmentRemoved( @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { + // query all the datasource schema, which includes, + // datasources explicitly marked for rebuilding + // datasources for the segments to be refreshed + // prebuilt datasources final Set dataSourcesToQuery = new HashSet<>(dataSourcesToRebuild); segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); + dataSourcesToQuery.addAll(tables.keySet()); + // Fetch datasource information from the Coordinator Map polledDataSourceMetadata = queryDataSourceInformation(dataSourcesToQuery); - // remove any extra datasources returned - polledDataSourceMetadata.keySet().removeIf(Predicates.not(dataSourcesToQuery::contains)); - // update datasource metadata in the cache polledDataSourceMetadata.forEach(this::updateDSMetadata); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 7436ecc07076..08741cc0f15d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -73,6 +73,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import java.io.IOException; import java.util.ArrayList; @@ -194,10 +196,10 @@ public void refresh( } /** - * Test the case when coordinator returns information for all the requested dataSources + * Test the case when coordinator returns information for all the requested datasources. */ @Test - public void testGetAllDsSchemaFromCoordinator() throws InterruptedException + public void testCoordinatorReturnsAllDSSchema() throws InterruptedException { final RowSignature dataSource1RowSignature = new QueryableIndexStorageAdapter(index1).getRowSignature(); final RowSignature dataSource2RowSignature = new QueryableIndexStorageAdapter(index2).getRowSignature(); @@ -218,6 +220,7 @@ public ListenableFuture> fetchDataSourceInformation( } }; + // don't expect any segment metadata queries QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( @@ -243,11 +246,11 @@ public ListenableFuture> fetchDataSourceInformation( } /** - * Test the case when Coordinator returns information for a subset of dataSources. - * Check if SegmentMetadataQuery is fired for segments of the remaining dataSources. + * Test the case when Coordinator returns information for a subset of datasources. + * Check if SegmentMetadataQuery is fired for segments of the remaining datasources. */ @Test - public void testGetFewDsSchemaFromCoordinator() throws InterruptedException + public void testCoordinatorReturnsFewDSSchema() throws InterruptedException { final RowSignature dataSource1RowSignature = new QueryableIndexStorageAdapter(index1).getRowSignature(); final RowSignature dataSource2RowSignature = new QueryableIndexStorageAdapter(index2).getRowSignature(); @@ -302,6 +305,40 @@ public ListenableFuture> fetchDataSourceInformation( EasyMock.verify(factoryMock, lifecycleMock); } + /** + * Verify that broker polls schema for all datasources in every cycle. + */ + @Test + public void testBrokerPollsAllDSSchema() throws InterruptedException + { + ArgumentCaptor> argumentCaptor = ArgumentCaptor.forClass(Set.class); + CoordinatorClient coordinatorClient = Mockito.mock(CoordinatorClient.class); + Mockito.when(coordinatorClient.fetchDataSourceInformation(argumentCaptor.capture())).thenReturn(Futures.immediateFuture(null)); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), + coordinatorClient + ); + + schema.start(); + schema.awaitInitialization(); + + Assert.assertEquals(Sets.newHashSet(DATASOURCE1, DATASOURCE2, DATASOURCE3, SOME_DATASOURCE), argumentCaptor.getValue()); + + refreshLatch = new CountDownLatch(1); + serverView.addSegment(newSegment("xyz", 0), ServerType.HISTORICAL); + + refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS); + + // verify that previously refreshed are included in the last coordinator poll + Assert.assertEquals(Sets.newHashSet(DATASOURCE1, DATASOURCE2, DATASOURCE3, SOME_DATASOURCE, "xyz"), argumentCaptor.getValue()); + } + @Test public void testGetTableMap() throws InterruptedException { From a4e2097ea85115364229c79a601c8c1a5f9153b7 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 26 Oct 2023 06:45:16 +0530 Subject: [PATCH 71/82] Address review comments --- docs/configuration/index.md | 2 +- .../druid/client/CachingClusteredClient.java | 3 ++ .../druid/client/CoordinatorServerView.java | 3 ++ .../apache/druid/client/SegmentLoadInfo.java | 10 ++++-- .../AbstractSegmentMetadataCache.java | 8 ++--- .../CoordinatorSegmentMetadataCache.java | 4 +-- ...inatorSegmentDataCacheConcurrencyTest.java | 12 +++---- .../CoordinatorSegmentMetadataCacheTest.java | 4 +-- .../schema/BrokerSegmentMetadataCache.java | 35 +++++++++++-------- .../BrokerSegmentMetadataCacheConfig.java | 2 +- ...erSegmentMetadataCacheConcurrencyTest.java | 12 +++---- .../BrokerSegmentMetadataCacheTest.java | 4 +-- 12 files changed, 57 insertions(+), 42 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 8c7122bedd53..46a79438ea5a 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -2003,7 +2003,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.useApproximateTopN`|Whether to use approximate [TopN queries](../querying/topnquery.md) when a SQL query could be expressed as such. If false, exact [GroupBy queries](../querying/groupbyquery.md) will be used instead.|true| |`druid.sql.planner.requireTimeCondition`|Whether to require SQL to have filter conditions on __time column so that all generated native queries will have user specified intervals. If true, all queries without filter condition on __time column will fail|false| |`druid.sql.planner.sqlTimeZone`|Sets the default time zone for the server, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|UTC| -|`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|true| +|`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| |`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|true| diff --git a/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java b/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java index 3f1679f756e7..7654fa32de79 100644 --- a/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java @@ -845,6 +845,9 @@ private byte[] computeQueryCacheKeyWithJoin() } } + /** + * Helper class to build a new timeline of filtered segments. + */ public static class TimelineConverter> implements UnaryOperator> { private final Iterable specs; diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index f303211b3558..a282700b2339 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -59,6 +59,9 @@ public class CoordinatorServerView implements InventoryView private final Object lock = new Object(); private final Map segmentLoadInfos; private final Map> timelines; + + // Map of server and QueryRunner. This is updated when a segment is added/removed. + // In parallel, it is used by {@link org.apache.druid.segment.metadata.SegmentMetadataQuerySegmentWalker} to run queries. private final ConcurrentMap serverQueryRunners; private final ConcurrentMap timelineCallbacks; private final ServerInventoryView baseView; diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index 2db810d6313e..8355ba501354 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -43,12 +43,16 @@ public SegmentLoadInfo(DataSegment segment) public boolean addServer(DruidServerMetadata server) { - return servers.add(server); + synchronized (this) { + return servers.add(server); + } } public boolean removeServer(DruidServerMetadata server) { - return servers.remove(server); + synchronized (this) { + return servers.remove(server); + } } public boolean isEmpty() @@ -62,7 +66,7 @@ public ImmutableSegmentLoadInfo toImmutableSegmentLoadInfo() } /** - * Randomly return one server from the sets of {@code servers} + * Randomly return one server from the sets of {@code servers}. */ public DruidServerMetadata pickOne() { diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index 56eda8000199..40366ccf8657 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -89,9 +89,7 @@ /** * An abstract class that listens for segment change events and caches segment metadata. It periodically refreshes * the segments, by fetching their metadata which includes schema information from sources like - * data nodes, db (the logic is specificed in the child class) and builds table schema. - * - *

    This class is generic and is parameterized by a type {@code T} that extends {@link DataSourceInformation}.

    + * data nodes, tasks, metadata database and builds table schema. * *

    This class has an abstract method {@link #refresh(Set, Set)} which the child class must override * with the logic to build and cache table schema.

    @@ -132,7 +130,7 @@ public abstract class AbstractSegmentMetadataCache refreshSegmentsForDataSource(final String dataSource, fin @VisibleForTesting @Nullable - public RowSignature buildDruidTable(final String dataSource) + public RowSignature buildDataSourceRowSignature(final String dataSource) { ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index 158ec92031ad..733ae2d44888 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -132,9 +132,9 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the datasources. for (String dataSource : dataSourcesToRebuild) { - final RowSignature rowSignature = buildDruidTable(dataSource); + final RowSignature rowSignature = buildDataSourceRowSignature(dataSource); if (rowSignature == null) { - log.info("datasource [%s] no longer exists, all metadata removed.", dataSource); + log.info("RowSignature null for dataSource [%s], implying it no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java index fd4696c315f9..1e0882fcf6aa 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java @@ -161,7 +161,7 @@ public void tearDown() throws Exception * This tests the contention between three components, {@link AbstractSegmentMetadataCache}, * {@code InventoryView}, and {@link BrokerServerView}. It first triggers * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with - * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDataSourceRowSignature} * is overridden to sleep before doing real work. While refreshing * {@code SegmentMetadataCache}, more new segments are added to * {@code InventoryView}, which triggers updates of {@code BrokerServerView}. @@ -183,7 +183,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV ) { @Override - public RowSignature buildDruidTable(final String dataSource) + public RowSignature buildDataSourceRowSignature(final String dataSource) { doInLock(() -> { try { @@ -194,7 +194,7 @@ public RowSignature buildDruidTable(final String dataSource) throw new RuntimeException(e); } }); - return super.buildDruidTable(dataSource); + return super.buildDataSourceRowSignature(dataSource); } }; @@ -269,7 +269,7 @@ public CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegme * {@link AbstractSegmentMetadataCache#refresh} and * {@link AbstractSegmentMetadataCache#getSegmentMetadataSnapshot()}. It first triggers * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with - * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDataSourceRowSignature} * is overridden to sleep before doing real work. While refreshing * {@code SegmentMetadataCache}, {@code getSegmentMetadataSnapshot()} is continuously * called to mimic reading the segments table of SystemSchema. All these calls @@ -289,7 +289,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() ) { @Override - public RowSignature buildDruidTable(final String dataSource) + public RowSignature buildDataSourceRowSignature(final String dataSource) { doInLock(() -> { try { @@ -300,7 +300,7 @@ public RowSignature buildDruidTable(final String dataSource) throw new RuntimeException(e); } }); - return super.buildDruidTable(dataSource); + return super.buildDataSourceRowSignature(dataSource); } }; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java index 26171366d5ac..a5eb8251acc7 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java @@ -114,9 +114,9 @@ public CoordinatorSegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetad ) { @Override - public RowSignature buildDruidTable(String dataSource) + public RowSignature buildDataSourceRowSignature(String dataSource) { - RowSignature table = super.buildDruidTable(dataSource); + RowSignature table = super.buildDataSourceRowSignature(dataSource); buildTableLatch.countDown(); return table; } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 825ad879078d..08d9fa409aae 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -33,6 +33,7 @@ import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.security.Escalator; @@ -43,6 +44,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -154,7 +156,7 @@ public ServerView.CallbackAction serverSegmentRemoved( @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { - // query all the datasource schema, which includes, + // query schema for all datasources, which includes, // datasources explicitly marked for rebuilding // datasources for the segments to be refreshed // prebuilt datasources @@ -195,7 +197,7 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the datasources. for (String dataSource : dataSourcesToRebuild) { - final RowSignature rowSignature = buildDruidTable(dataSource); + final RowSignature rowSignature = buildDataSourceRowSignature(dataSource); if (rowSignature == null) { log.info("datasource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); @@ -209,21 +211,14 @@ public void refresh(final Set segmentsToRefresh, final Set da private Map queryDataSourceInformation(Set dataSourcesToQuery) { - final Map polledDataSourceMetadata = new HashMap<>(); - Stopwatch stopwatch = Stopwatch.createStarted(); + List dataSourceInformations = null; + + emitter.emit(ServiceMetricEvent.builder().setMetric( + "metadatacache/schemaPoll/count", 1)); try { - emitter.emit(ServiceMetricEvent.builder().setMetric( - "metadatacache/schemaPoll/count", 1)); - FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) - .forEach(dataSourceInformation -> polledDataSourceMetadata.put( - dataSourceInformation.getDataSource(), - dataSourceMetadataFactory.build( - dataSourceInformation.getDataSource(), - dataSourceInformation.getRowSignature() - ) - )); + dataSourceInformations = FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true); } catch (Exception e) { log.warn(e, "Failed to query datasource information from the Coordinator."); @@ -235,6 +230,18 @@ private Map queryDataSourceInformation(Set polledDataSourceMetadata = new HashMap<>(); + + if (null != dataSourceInformations) { + dataSourceInformations.forEach(dataSourceInformation -> polledDataSourceMetadata.put( + dataSourceInformation.getDataSource(), + dataSourceMetadataFactory.build( + dataSourceInformation.getDataSource(), + dataSourceInformation.getRowSignature() + ) + )); + } + return polledDataSourceMetadata; } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index b66939954157..d6842e6a896d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -40,7 +40,7 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig { // A flag indicating whether to cache polled segments from the Coordinator. @JsonProperty - private boolean metadataSegmentCacheEnable = true; + private boolean metadataSegmentCacheEnable = false; // Interval for polling segments from the coordinator. @JsonProperty diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java index 3b189821e999..e1119dc36ee5 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConcurrencyTest.java @@ -120,7 +120,7 @@ public void tearDown() throws Exception * This tests the contention between three components, {@link AbstractSegmentMetadataCache}, * {@code InventoryView}, and {@link BrokerServerView}. It first triggers * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with - * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDataSourceRowSignature} * is overridden to sleep before doing real work. While refreshing * {@code SegmentMetadataCache}, more new segments are added to * {@code InventoryView}, which triggers updates of {@code BrokerServerView}. @@ -144,7 +144,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV ) { @Override - public RowSignature buildDruidTable(final String dataSource) + public RowSignature buildDataSourceRowSignature(final String dataSource) { doInLock(() -> { try { @@ -155,7 +155,7 @@ public RowSignature buildDruidTable(final String dataSource) throw new RuntimeException(e); } }); - return super.buildDruidTable(dataSource); + return super.buildDataSourceRowSignature(dataSource); } }; @@ -232,7 +232,7 @@ public ServerView.CallbackAction serverSegmentRemoved(DruidServerMetadata server * {@link AbstractSegmentMetadataCache#refresh} and * {@link AbstractSegmentMetadataCache#getSegmentMetadataSnapshot()}. It first triggers * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with - * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDataSourceRowSignature} * is overridden to sleep before doing real work. While refreshing * {@code SegmentMetadataCache}, {@code getSegmentMetadataSnapshot()} is continuously * called to mimic reading the segments table of SystemSchema. All these calls @@ -254,7 +254,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() ) { @Override - public RowSignature buildDruidTable(final String dataSource) + public RowSignature buildDataSourceRowSignature(final String dataSource) { doInLock(() -> { try { @@ -265,7 +265,7 @@ public RowSignature buildDruidTable(final String dataSource) throw new RuntimeException(e); } }); - return super.buildDruidTable(dataSource); + return super.buildDataSourceRowSignature(dataSource); } }; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 08741cc0f15d..488478e2d998 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -138,9 +138,9 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMeta ) { @Override - public RowSignature buildDruidTable(String dataSource) + public RowSignature buildDataSourceRowSignature(String dataSource) { - RowSignature table = super.buildDruidTable(dataSource); + RowSignature table = super.buildDataSourceRowSignature(dataSource); buildTableLatch.countDown(); return table; } From 902abd3939db35ecc1dcae1e6ae9288c7605ecbc Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 26 Oct 2023 07:15:32 +0530 Subject: [PATCH 72/82] Run DruidSchemaInternRowSignatureBenchmark using BrokerSegmentMetadataCache for cache --- ...ruidSchemaInternRowSignatureBenchmark.java | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 570861052e7a..dbf9d39c2a17 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -22,21 +22,25 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.SegmentManager; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Escalator; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; +import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataFactory; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.LinearShardSpec; @@ -68,22 +72,26 @@ public class DruidSchemaInternRowSignatureBenchmark { private SegmentMetadataCacheForBenchmark cache; - private static class SegmentMetadataCacheForBenchmark extends CoordinatorSegmentMetadataCache + private static class SegmentMetadataCacheForBenchmark extends BrokerSegmentMetadataCache { public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, - final CoordinatorServerView serverView, + final TimelineServerView serverView, + final SegmentManager segmentManager, + final JoinableFactory joinableFactory, final Escalator escalator, - final InternalQueryConfig internalQueryConfig + final InternalQueryConfig brokerInternalQueryConfig ) { super( queryLifecycleFactory, serverView, - SegmentMetadataCacheConfig.create(), + BrokerSegmentMetadataCacheConfig.create(), escalator, - internalQueryConfig, - new NoopServiceEmitter() + brokerInternalQueryConfig, + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataFactory(joinableFactory, segmentManager), + new NoopCoordinatorClient() ); } @@ -167,10 +175,13 @@ public void setup() { cache = new SegmentMetadataCacheForBenchmark( EasyMock.mock(QueryLifecycleFactory.class), - EasyMock.mock(CoordinatorServerView.class), + EasyMock.mock(TimelineServerView.class), + null, + null, null, null ); + DruidServerMetadata serverMetadata = new DruidServerMetadata( "dummy", "dummy", From bcab458d6d6b821af56fa3c910ae7c9a64a0160c Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 26 Oct 2023 18:12:25 +0530 Subject: [PATCH 73/82] Address feedback --- .../druid/client/CachingClusteredClient.java | 22 +++++++++---------- .../druid/client/CoordinatorServerView.java | 4 ++-- .../apache/druid/client/SegmentLoadInfo.java | 2 +- .../SegmentMetadataQuerySegmentWalker.java | 2 +- .../CoordinatorSegmentMetadataCacheTest.java | 16 +++++++------- .../metadata/SegmentMetadataCacheCommon.java | 8 ++----- ...SegmentMetadataQuerySegmentWalkerTest.java | 2 +- .../metadata/TestCoordinatorServerView.java | 6 ++--- .../schema/BrokerSegmentMetadataCache.java | 2 +- .../BrokerSegmentMetadataCacheConfig.java | 1 + .../BrokerSegmentMetadataCacheConfigTest.java | 4 ++-- .../BrokerSegmentMetadataCacheTest.java | 2 +- 12 files changed, 34 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java b/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java index 7654fa32de79..70baeb239d5b 100644 --- a/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/org/apache/druid/client/CachingClusteredClient.java @@ -848,7 +848,7 @@ private byte[] computeQueryCacheKeyWithJoin() /** * Helper class to build a new timeline of filtered segments. */ - public static class TimelineConverter> implements UnaryOperator> + public static class TimelineConverter> implements UnaryOperator> { private final Iterable specs; @@ -858,29 +858,29 @@ public TimelineConverter(final Iterable specs) } @Override - public TimelineLookup apply(TimelineLookup timeline) + public TimelineLookup apply(TimelineLookup timeline) { - final VersionedIntervalTimeline timeline2 = - new VersionedIntervalTimeline<>(Ordering.natural(), true); - Iterator> unfilteredIterator = + Iterator> unfilteredIterator = Iterators.transform(specs.iterator(), spec -> toChunkEntry(timeline, spec)); - Iterator> iterator = Iterators.filter( + Iterator> iterator = Iterators.filter( unfilteredIterator, Objects::nonNull ); + final VersionedIntervalTimeline newTimeline = + new VersionedIntervalTimeline<>(Ordering.natural(), true); // VersionedIntervalTimeline#addAll implementation is much more efficient than calling VersionedIntervalTimeline#add // in a loop when there are lot of segments to be added for same interval and version. - timeline2.addAll(iterator); - return timeline2; + newTimeline.addAll(iterator); + return newTimeline; } @Nullable - private PartitionChunkEntry toChunkEntry( - TimelineLookup timeline, + private PartitionChunkEntry toChunkEntry( + TimelineLookup timeline, SegmentDescriptor spec ) { - PartitionChunk chunk = timeline.findChunk( + PartitionChunk chunk = timeline.findChunk( spec.getInterval(), spec.getVersion(), spec.getPartitionNumber() diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index a282700b2339..e79e68e726fb 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -181,9 +181,9 @@ private void serverAddedSegment(final DruidServerMetadata server, final DataSegm segmentLoadInfos.put(segmentId, segmentLoadInfo); } - if (null != druidClientFactory) { + if (druidClientFactory != null) { QueryRunner queryRunner = serverQueryRunners.get(server.getName()); - if (null == queryRunner) { + if (queryRunner == null) { DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); if (inventoryValue == null) { log.warn( diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index 8355ba501354..110287e7537e 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -38,7 +38,7 @@ public SegmentLoadInfo(DataSegment segment) { Preconditions.checkNotNull(segment, "segment"); this.segment = segment; - this.servers = Sets.newConcurrentHashSet(); + this.servers = Sets.newHashSet(); } public boolean addServer(DruidServerMetadata server) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalker.java index 55a5a9b3fcc1..3de63c35e3a9 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalker.java @@ -135,7 +135,7 @@ private Sequence run( final TimelineLookup timeline = serverView.getTimeline( query.getDataSource() ); - if (null == timeline) { + if (timeline == null) { return Sequences.empty(); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java index a5eb8251acc7..22a5d7a67c41 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java @@ -424,7 +424,7 @@ public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException Assert.assertEquals(0L, currentMetadata.isRealtime()); DruidServer realtimeServer = druidServers.stream() - .filter(s -> s.getType().equals(ServerType.REALTIME)) + .filter(s -> s.getType().equals(ServerType.INDEXER_EXECUTOR)) .findAny() .orElse(null); Assert.assertNotNull(realtimeServer); @@ -506,7 +506,7 @@ public void addSegment(final DruidServerMetadata server, final DataSegment segme }; DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.REALTIME); + serverView.addSegment(segment, ServerType.INDEXER_EXECUTOR); serverView.addSegment(segment, ServerType.HISTORICAL); Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); @@ -550,7 +550,7 @@ public void addSegment(final DruidServerMetadata server, final DataSegment segme } }; - serverView.addSegment(newSegment(datasource, 1), ServerType.REALTIME); + serverView.addSegment(newSegment(datasource, 1), ServerType.INDEXER_EXECUTOR); Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); Assert.assertEquals(7, schema.getTotalSegments()); @@ -641,11 +641,11 @@ public void removeSegment(final DataSegment segment) }; DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.REALTIME); + serverView.addSegment(segment, ServerType.INDEXER_EXECUTOR); Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); schema.refresh(Sets.newHashSet(segment.getId()), Sets.newHashSet(datasource)); - serverView.removeSegment(segment, ServerType.REALTIME); + serverView.removeSegment(segment, ServerType.INDEXER_EXECUTOR); Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); Assert.assertEquals(6, schema.getTotalSegments()); @@ -700,12 +700,12 @@ public void removeSegment(final DataSegment segment) newSegment(datasource, 1), newSegment(datasource, 2) ); - serverView.addSegment(segments.get(0), ServerType.REALTIME); + serverView.addSegment(segments.get(0), ServerType.INDEXER_EXECUTOR); serverView.addSegment(segments.get(1), ServerType.HISTORICAL); Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); - serverView.removeSegment(segments.get(0), ServerType.REALTIME); + serverView.removeSegment(segments.get(0), ServerType.INDEXER_EXECUTOR); Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); Assert.assertEquals(7, schema.getTotalSegments()); @@ -1073,7 +1073,7 @@ public void removeSegment(final DataSegment segment) newSegment(dataSource, 2) ); serverView.addSegment(segments.get(0), ServerType.HISTORICAL); - serverView.addSegment(segments.get(1), ServerType.REALTIME); + serverView.addSegment(segments.get(1), ServerType.INDEXER_EXECUTOR); Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(dataSource)); diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 2fc04ba658fd..d421a15e35fc 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -22,7 +22,6 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -51,6 +50,7 @@ import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; +import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; @@ -61,7 +61,7 @@ import java.util.List; import java.util.Map; -public abstract class SegmentMetadataCacheCommon +public abstract class SegmentMetadataCacheCommon extends InitializedNullHandlingTest { public static final String DATASOURCE1 = "foo"; public static final String DATASOURCE2 = "foo2"; @@ -108,10 +108,6 @@ public abstract class SegmentMetadataCacheCommon public DataSegment segment4; public DataSegment segment5; - static { - NullHandling.initializeForTests(); - } - public void setUpCommon() { resourceCloser = Closer.create(); diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java index 7d5638b14d6a..d51454f0a5cb 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataQuerySegmentWalkerTest.java @@ -117,7 +117,7 @@ public long getMaxQueuedBytes() new DruidServer[]{ new DruidServer("test1", "test1", null, 10, ServerType.HISTORICAL, "bye", 0), new DruidServer("test2", "test2", null, 10, ServerType.HISTORICAL, "bye", 0), - new DruidServer("test3", "test2", null, 10, ServerType.REALTIME, "bye", 0) + new DruidServer("test3", "test2", null, 10, ServerType.INDEXER_EXECUTOR, "bye", 0) }; random = new Random(0); diff --git a/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java b/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java index 160b8b9c5e09..5d074652ee8f 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java @@ -66,7 +66,7 @@ public class TestCoordinatorServerView extends CoordinatorServerView "dummy2", null, 0, - ServerType.REALTIME, + ServerType.INDEXER_EXECUTOR, "dummy", 0 ); @@ -113,7 +113,7 @@ private DruidServer getServerForType(ServerType serverType) switch (serverType) { case BROKER: return DUMMY_BROKER; - case REALTIME: + case INDEXER_EXECUTOR: return DUMMY_SERVER_REALTIME; default: return DUMMY_SERVER; @@ -228,7 +228,7 @@ public List getSegmentsOfServer(DruidServer druidServer) { if (druidServer.getType() == ServerType.BROKER) { return Lists.newArrayList(brokerSegments); - } else if (druidServer.getType() == ServerType.REALTIME) { + } else if (druidServer.getType() == ServerType.INDEXER_EXECUTOR) { return Lists.newArrayList(realtimeSegments); } else { return Lists.newArrayList(segments); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 08d9fa409aae..07a98cc5c11e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -232,7 +232,7 @@ private Map queryDataSourceInformation(Set polledDataSourceMetadata = new HashMap<>(); - if (null != dataSourceInformations) { + if (dataSourceInformations != null) { dataSourceInformations.forEach(dataSourceInformation -> polledDataSourceMetadata.put( dataSourceInformation.getDataSource(), dataSourceMetadataFactory.build( diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index d6842e6a896d..6931f6ed33f1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -50,6 +50,7 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig @JsonProperty private boolean awaitInitializationOnStart = true; + // This is meant to be used when running ITs to disable table schema building in broker. @JsonProperty private boolean disableSegmentMetadataQueries = false; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java index a2f5ef4b5395..3efb9d1ea5a1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java @@ -50,7 +50,7 @@ public void testDefaultConfig() provider.inject(properties, injector.getInstance(JsonConfigurator.class)); final BrokerSegmentMetadataCacheConfig config = provider.get(); Assert.assertTrue(config.isAwaitInitializationOnStart()); - Assert.assertTrue(config.isMetadataSegmentCacheEnable()); + Assert.assertFalse(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); Assert.assertEquals(new AbstractSegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } @@ -73,7 +73,7 @@ public void testCustomizedConfig() provider.inject(properties, injector.getInstance(JsonConfigurator.class)); final BrokerSegmentMetadataCacheConfig config = provider.get(); Assert.assertFalse(config.isAwaitInitializationOnStart()); - Assert.assertTrue(config.isMetadataSegmentCacheEnable()); + Assert.assertFalse(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); Assert.assertEquals( new AbstractSegmentMetadataCache.FirstTypeMergePolicy(), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 488478e2d998..3cc566cfc1b0 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -860,7 +860,7 @@ public void removeSegment(final DataSegment segment) newSegment(dataSource, 2) ); serverView.addSegment(segments.get(0), ServerType.HISTORICAL); - serverView.addSegment(segments.get(1), ServerType.REALTIME); + serverView.addSegment(segments.get(1), ServerType.INDEXER_EXECUTOR); Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(dataSource)); From 32a4065a7d7a2924970426d1f93736b3c02b1f45 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 26 Oct 2023 18:20:41 +0530 Subject: [PATCH 74/82] Simplify logic for setting isRealtime in sys segments table --- .../java/org/apache/druid/sql/calcite/schema/SystemSchema.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index fa0b6babe12b..fb6bc002e2a4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -318,7 +318,7 @@ public Enumerable scan(DataContext root) numRows = val.getNumRows(); } - isRealtime = Boolean.TRUE.equals(val.isRealtime()) ? 1 : 0; + isRealtime = val.isRealtime() ? 1 : 0; // set of segments returned from Coordinator include published and realtime segments // so realtime segments are not published and vice versa From 6b04ee709a6d9d7d8b5f7dc36a5ec31fa998dea1 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 26 Oct 2023 18:24:59 +0530 Subject: [PATCH 75/82] Remove forbidden api invocation --- .../src/main/java/org/apache/druid/client/SegmentLoadInfo.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index 110287e7537e..628e3b6c021a 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -26,6 +26,7 @@ import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.Overshadowable; +import java.util.HashSet; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; @@ -38,7 +39,7 @@ public SegmentLoadInfo(DataSegment segment) { Preconditions.checkNotNull(segment, "segment"); this.segment = segment; - this.servers = Sets.newHashSet(); + this.servers = new HashSet<>(); } public boolean addServer(DruidServerMetadata server) From 152c480ac9fc3c454c80523a7ebe77a803cb5cbc Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 26 Oct 2023 19:32:44 +0530 Subject: [PATCH 76/82] Debug log when coordinator poll fails --- .../druid/sql/calcite/schema/BrokerSegmentMetadataCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 07a98cc5c11e..a52c6d892595 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -221,7 +221,7 @@ private Map queryDataSourceInformation(Set Date: Fri, 27 Oct 2023 11:21:37 +0530 Subject: [PATCH 77/82] Fix CoordinatorSegmentMetadataCacheTest --- .../main/java/org/apache/druid/client/SegmentLoadInfo.java | 1 - .../druid/segment/metadata/TestCoordinatorServerView.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index 628e3b6c021a..3f1fa559abc0 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -21,7 +21,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; -import com.google.common.collect.Sets; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.Overshadowable; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java b/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java index 5d074652ee8f..3deec71b64dc 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/TestCoordinatorServerView.java @@ -122,7 +122,7 @@ private DruidServer getServerForType(ServerType serverType) private void addToTimeline(DataSegment dataSegment, DruidServer druidServer) { - if (druidServer.getMetadata().getType() == ServerType.REALTIME) { + if (druidServer.getMetadata().getType() == ServerType.INDEXER_EXECUTOR) { realtimeSegments.add(dataSegment); } else if (druidServer.getMetadata().getType() == ServerType.BROKER) { brokerSegments.add(dataSegment); @@ -183,7 +183,7 @@ public void removeSegment(DataSegment segment, ServerType serverType) if (serverType == ServerType.BROKER) { druidServerMetadata = DUMMY_BROKER.getMetadata(); brokerSegments.remove(segment); - } else if (serverType == ServerType.REALTIME) { + } else if (serverType == ServerType.INDEXER_EXECUTOR) { druidServerMetadata = DUMMY_SERVER_REALTIME.getMetadata(); realtimeSegments.remove(segment); } else { From bb69cde84408b6b869e359ba0d0cae22dee5f094 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 28 Oct 2023 18:24:51 +0530 Subject: [PATCH 78/82] Minor changes --- .../druid/client/coordinator/CoordinatorClientImpl.java | 4 ---- .../druid/segment/metadata/AbstractSegmentMetadataCache.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index 7cd9906b16bd..93c22bbdbff8 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -37,7 +37,6 @@ import org.joda.time.Interval; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Set; @@ -147,9 +146,6 @@ public ListenableFuture> fetchUsedSegments(String dataSource, public ListenableFuture> fetchDataSourceInformation(Set dataSources) { final String path = "/druid/coordinator/v1/metadata/dataSourceInformation"; - if (null == dataSources) { - dataSources = new HashSet<>(); - } return FutureUtils.transform( client.asyncRequest( new RequestBuilder(HttpMethod.POST, path) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index 40366ccf8657..bcc08f542748 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -344,7 +344,7 @@ private void setInitializedAndReportInitTime(Stopwatch stopwatch) @LifecycleStart public void start() throws InterruptedException { - log.info("%s waiting for initialization.", getClass().getSimpleName()); + log.info("%s starting cache initialization.", getClass().getSimpleName()); startCacheExec(); if (config.isAwaitInitializationOnStart()) { From bd6a5ef9690003da38e5859982bc038979d04b62 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 31 Oct 2023 13:32:02 +0530 Subject: [PATCH 79/82] Synchronisation in SegmentLoadInfo --- .../java/org/apache/druid/client/SegmentLoadInfo.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index 3f1fa559abc0..2d2212a02a76 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -57,12 +57,16 @@ public boolean removeServer(DruidServerMetadata server) public boolean isEmpty() { - return servers.isEmpty(); + synchronized (this) { + return servers.isEmpty(); + } } public ImmutableSegmentLoadInfo toImmutableSegmentLoadInfo() { - return new ImmutableSegmentLoadInfo(segment, servers); + synchronized (this) { + return new ImmutableSegmentLoadInfo(segment, servers); + } } /** From 23ff74029683e2ed3fca33ee00b509f651520df4 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 1 Nov 2023 00:57:17 +0530 Subject: [PATCH 80/82] Add comments --- docs/api-reference/legacy-metadata-api.md | 4 ++-- .../org/apache/druid/server/http/MetadataResource.java | 7 ++++--- .../druid/sql/calcite/schema/MetadataSegmentView.java | 2 ++ .../apache/druid/sql/calcite/schema/SystemSchema.java | 10 +++++----- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/api-reference/legacy-metadata-api.md b/docs/api-reference/legacy-metadata-api.md index b573816fe23f..baf9418f5caf 100644 --- a/docs/api-reference/legacy-metadata-api.md +++ b/docs/api-reference/legacy-metadata-api.md @@ -118,7 +118,7 @@ Returns a list of all segments for each datasource with the full segment metadat `GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments` -Additionally, returns the realtime segments for all datasources, with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. +Returns a list of all published and realtime segments for each datasource with the full segment metadata and extra fields `overshadowed`,`realtime` & `numRows`. Realtime segments are returned only when `druid.coordinator.centralizedTableSchema.enabled` is set on the Coordinator. `GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&datasources={dataSourceName1}&datasources={dataSourceName2}` @@ -126,7 +126,7 @@ Returns a list of all segments for one or more specific datasources with the ful `GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments&datasources={dataSourceName1}&datasources={dataSourceName2}` -Additionally, returns the realtime segments for the specified datasources, with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. +Returns a list of all published and realtime segments for the specified datasources with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. Realtime segments are returned only when `druid.coordinator.centralizedTableSchema.enabled` is set on the Coordinator. `GET /druid/coordinator/v1/metadata/datasources` diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index c300cd13504a..425ee8007be7 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -156,6 +156,7 @@ public Response getAllUsedSegments( } if (includeOvershadowedStatus != null) { + // note that realtime segments are returned only when druid.coordinator.centralizedTableSchema.enabled is set on the Coordinator return getAllUsedSegmentsWithAdditionalDetails(req, dataSources, includeRealtimeSegments); } @@ -208,7 +209,7 @@ private Response getAllUsedSegmentsWithAdditionalDetails( : coordinator.getReplicationFactor(segment.getId()); Long numRows = null; - if (null != coordinatorSegmentMetadataCache) { + if (coordinatorSegmentMetadataCache != null) { AvailableSegmentMetadata availableSegmentMetadata = coordinatorSegmentMetadataCache.getAvailableSegmentMetadata( segment.getDataSource(), segment.getId() @@ -231,7 +232,7 @@ private Response getAllUsedSegmentsWithAdditionalDetails( Stream finalSegments = segmentStatus; // conditionally add realtime segments information - if (null != includeRealtimeSegments && null != coordinatorSegmentMetadataCache) { + if (includeRealtimeSegments != null && coordinatorSegmentMetadataCache != null) { final Stream realtimeSegmentStatus = coordinatorSegmentMetadataCache .getSegmentMetadataSnapshot() .values() @@ -376,7 +377,7 @@ public Response getDataSourceInformation( ) { // if {@code coordinatorSegmentMetadataCache} is null, implies the feature is disabled. Return NOT_FOUND. - if (null == coordinatorSegmentMetadataCache) { + if (coordinatorSegmentMetadataCache == null) { return Response.status(Response.Status.NOT_FOUND).build(); } Map dataSourceSchemaMap = coordinatorSegmentMetadataCache.getDataSourceInformationMap(); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java index 50a506d0e6f3..9bbe1491a324 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java @@ -193,6 +193,8 @@ private JsonParserIterator getMetadataSegments( Set watchedDataSources ) { + // includeRealtimeSegments flag would additionally request realtime segments + // note that realtime segments are returned only when druid.coordinator.centralizedTableSchema.enabled is set on the Coordinator StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments"); if (watchedDataSources != null && !watchedDataSources.isEmpty()) { log.debug( diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index fb6bc002e2a4..ff242d58e601 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -276,7 +276,7 @@ public TableType getJdbcTableType() @Override public Enumerable scan(DataContext root) { - //get available segments from druidSchema + // get available segments from druidSchema final Map availableSegmentMetadata = druidSchema.cache().getSegmentMetadataSnapshot(); final Iterator> availableSegmentEntries = @@ -311,9 +311,9 @@ public Enumerable scan(DataContext root) numRows = partialSegmentData.getNumRows(); } - // If table schema building is enabled on the Coordinator, SegmentMetadataCache on the + // If druid.coordinator.centralizedTableSchema.enabled is set on the Coordinator, SegmentMetadataCache on the // broker might have outdated or no information regarding numRows and rowSignature for a segment. - // We should use {@code numRows} from the segment polled from the coordinator. + // In that case, we should use {@code numRows} from the segment polled from the coordinator. if (null != val.getNumRows()) { numRows = val.getNumRows(); } @@ -349,7 +349,7 @@ public Enumerable scan(DataContext root) segment.getLastCompactionState() == null ? null : jsonMapper.writeValueAsString(segment.getLastCompactionState()), // If the segment is unpublished, we won't have this information yet. // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. - // This should be automatically updated in the next refesh with Coordinator. + // This should be automatically updated in the next Coordinator poll. val.getReplicationFactor() == null ? REPLICATION_FACTOR_UNKNOWN : (long) val.getReplicationFactor() }; } @@ -358,7 +358,7 @@ public Enumerable scan(DataContext root) } }); - // When table schema building is enabled on the Coordinator, all the segments in this loop + // If druid.coordinator.centralizedTableSchema.enabled is set on the Coordinator, all the segments in this loop // would be covered in the previous iteration since Coordinator would return realtime segments as well. final FluentIterable availableSegments = FluentIterable .from(() -> getAuthorizedAvailableSegments( From 3de534f723e9a2288fb197270e75125010c7d22e Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 1 Nov 2023 18:58:50 +0530 Subject: [PATCH 81/82] Remove explicit synchronisation from SegmentLoadInfo#pickOne --- .../apache/druid/client/SegmentLoadInfo.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index 2d2212a02a76..9d25a70a4f64 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -20,12 +20,12 @@ package org.apache.druid.client; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterators; +import com.google.common.collect.Sets; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.Overshadowable; -import java.util.HashSet; +import java.util.Iterator; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; @@ -38,35 +38,27 @@ public SegmentLoadInfo(DataSegment segment) { Preconditions.checkNotNull(segment, "segment"); this.segment = segment; - this.servers = new HashSet<>(); + this.servers = Sets.newConcurrentHashSet(); } public boolean addServer(DruidServerMetadata server) { - synchronized (this) { - return servers.add(server); - } + return servers.add(server); } public boolean removeServer(DruidServerMetadata server) { - synchronized (this) { - return servers.remove(server); - } + return servers.remove(server); } public boolean isEmpty() { - synchronized (this) { - return servers.isEmpty(); - } + return servers.isEmpty(); } public ImmutableSegmentLoadInfo toImmutableSegmentLoadInfo() { - synchronized (this) { - return new ImmutableSegmentLoadInfo(segment, servers); - } + return new ImmutableSegmentLoadInfo(segment, servers); } /** @@ -74,9 +66,21 @@ public ImmutableSegmentLoadInfo toImmutableSegmentLoadInfo() */ public DruidServerMetadata pickOne() { - synchronized (this) { - return Iterators.get(servers.iterator(), ThreadLocalRandom.current().nextInt(servers.size())); + int randomPosition = ThreadLocalRandom.current().nextInt(servers.size()); + + Iterator serversIterator = servers.iterator(); + int index = 0; + DruidServerMetadata randomServer = null; + + while (serversIterator.hasNext()) { + randomServer = serversIterator.next(); + if (index == randomPosition) { + break; + } + index++; } + + return randomServer; } @Override From 571af6474a08b72a90a7d0994ffdf17ebf9a2571 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 3 Nov 2023 14:24:29 +0530 Subject: [PATCH 82/82] Minor changes --- .../src/main/java/org/apache/druid/client/SegmentLoadInfo.java | 2 +- .../segment/metadata/CoordinatorSegmentMetadataCache.java | 2 +- .../java/org/apache/druid/server/http/MetadataResource.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java index 9d25a70a4f64..fe32490a3e03 100644 --- a/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java +++ b/server/src/main/java/org/apache/druid/client/SegmentLoadInfo.java @@ -62,7 +62,7 @@ public ImmutableSegmentLoadInfo toImmutableSegmentLoadInfo() } /** - * Randomly return one server from the sets of {@code servers}. + * Randomly return one server from the set of {@code servers}. */ public DruidServerMetadata pickOne() { diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index 733ae2d44888..960921c9e93b 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -107,7 +107,7 @@ public ServerView.CallbackAction serverSegmentRemoved( } /** - * Fires SegmentMetadataQuery to fetch schema information for each segment in the refresh list. + * Executes SegmentMetadataQuery to fetch schema information for each segment in the refresh list. * The schema information for individual segments is combined to construct a table schema, which is then cached. * * @param segmentsToRefresh segments for which the schema might have changed diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 425ee8007be7..bd12e99dc2b5 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -157,6 +157,7 @@ public Response getAllUsedSegments( if (includeOvershadowedStatus != null) { // note that realtime segments are returned only when druid.coordinator.centralizedTableSchema.enabled is set on the Coordinator + // when the feature is disabled we do not want to increase the payload size polled by the Brokers, since they already have this information return getAllUsedSegmentsWithAdditionalDetails(req, dataSources, includeRealtimeSegments); } @@ -363,7 +364,7 @@ public Response getSegment( } /** - * API to fetch {@link DataSourceInformation} for the specified dataSources. + * API to fetch {@link DataSourceInformation} for the specified datasources. * * @param dataSources list of dataSources to be queried * @return information including schema details for the specified datasources