Merge branch 'master' of github.com:apache/druid into add_audit_logger

.github/workflows/standard-its.yml

@@ -77,7 +77,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        testing_group: [query, query-retry, query-error, security, high-availability, centralized-table-schema]
+        testing_group: [query, query-retry, query-error, security, high-availability, centralized-datasource-schema]
     uses: ./.github/workflows/reusable-standard-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
@@ -195,6 +195,6 @@ jobs:
     with:
       build_jdk: 8
       runtime_jdk: 8
-      testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-table-schema
+      testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties,centralized-datasource-schema
       use_indexer: ${{ matrix.indexer }}
       group: other

distribution/bin/check-licenses.py

@@ -202,8 +202,19 @@ def set_attr(self, data):
 
     def set_license(self, data):
         if data.upper().find("GPL") < 0:
-            if self.license != 'Apache License version 2.0':
-                self.license = self.compatible_license_names[data]
+            # Check if the license assosciated with the component is acccepted
+            # set_license() will pick the first acceptable license
+            # this fixes issue where a multi-licensed component
+            # could override accepted license with not accepted one
+            # e.g., EPL / GPL for logback-core
+            if self.license not in self.compatible_license_names:
+                try:
+                    self.license = self.compatible_license_names[data]
+                except KeyError:
+                    print("Unsupported license: " + data)
+                    print("For:" +  self.group_id + " "  + self.artifact_id + " in: "+ self.druid_module_name)
+            else:
+                print(self.group_id + " "  + self.artifact_id + " in: " + self.druid_module_name + " with: " + self.license + " ignoring " + data)
 
 
 def print_log_to_stderr(string):

docs/api-reference/legacy-metadata-api.md

@@ -118,15 +118,15 @@ Returns a list of all segments for each datasource with the full segment metadat
 
 `GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments`
 
-Returns a list of all published and realtime segments for each datasource with the full segment metadata and extra fields `overshadowed`,`realtime` & `numRows`. Realtime segments are returned only when `druid.coordinator.centralizedTableSchema.enabled` is set on the Coordinator. 
+Returns a list of all published and realtime segments for each datasource with the full segment metadata and extra fields `overshadowed`,`realtime` & `numRows`. Realtime segments are returned only when `druid.centralizedDatasourceSchema.enabled` is set on the Coordinator. 
 
 `GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&datasources={dataSourceName1}&datasources={dataSourceName2}`
 
 Returns a list of all segments for one or more specific datasources with the full segment metadata and an extra field `overshadowed`.
 
 `GET /druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments&datasources={dataSourceName1}&datasources={dataSourceName2}`
 
-Returns a list of all published and realtime segments for the specified datasources with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. Realtime segments are returned only when `druid.coordinator.centralizedTableSchema.enabled` is set on the Coordinator.
+Returns a list of all published and realtime segments for the specified datasources with the full segment metadata and extra fields `overshadwed`,`realtime` & `numRows`. Realtime segments are returned only when `druid.centralizedDatasourceSchema.enabled` is set on the Coordinator.
 
 `GET /druid/coordinator/v1/metadata/datasources`
 

docs/configuration/index.md

@@ -876,7 +876,7 @@ These Coordinator static configurations can be defined in the `coordinator/runti
 |`druid.coordinator.loadqueuepeon.repeatDelay`|The start and repeat delay for the `loadqueuepeon`, which manages the load and drop of segments.|`PT0.050S` (50 ms)|
 |`druid.coordinator.asOverlord.enabled`|Boolean value for whether this Coordinator service should act like an Overlord as well. This configuration allows users to simplify a Druid cluster by not having to deploy any standalone Overlord services. If set to true, then Overlord console is available at `http://coordinator-host:port/console.html` and be sure to set `druid.coordinator.asOverlord.overlordService` also.|false|
 |`druid.coordinator.asOverlord.overlordService`| Required, if `druid.coordinator.asOverlord.enabled` is `true`. This must be same value as `druid.service` on standalone Overlord services and `druid.selectors.indexing.serviceName` on Middle Managers.|NULL|
-|`druid.coordinator.centralizedTableSchema.enabled`|Boolean flag for enabling table schema building on the Coordinator.|false|
+|`druid.centralizedDatasourceSchema.enabled`|Boolean flag for enabling datasource schema building on the Coordinator.|false|
 
 ##### Metadata management
 

extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorCollector.java

@@ -91,8 +91,11 @@ void fold(@Nullable VarianceAggregatorCollector other)
     this.sum += other.sum;
   }
 
-  static Object combineValues(Object lhs, @Nullable Object rhs)
+  static Object combineValues(@Nullable Object lhs, @Nullable Object rhs)
   {
+    if (lhs == null) {
+      return rhs;
+    }
     ((VarianceAggregatorCollector) lhs).fold((VarianceAggregatorCollector) rhs);
     return lhs;
   }

extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java

@@ -173,6 +173,14 @@ public void testVarianceComparatorShouldCompareSumIfVarianceAndCountsAreEqual()
     Assert.assertEquals(0, VarianceAggregatorCollector.COMPARATOR.compare(v1, v2));
   }
 
+  @Test
+  public void testNullCollectors()
+  {
+    VarianceAggregatorCollector collector =
+        (VarianceAggregatorCollector) VarianceAggregatorCollector.combineValues(null, null);
+    Assert.assertNull(collector);
+  }
+
   private static class FloatHandOver extends TestFloatColumnSelector
   {
     float v;

integration-tests/docker/docker-compose.centralized-table-schema.yml → integration-tests/docker/docker-compose.centralized-datasource-schema.yml

@@ -35,7 +35,7 @@ services:
       service: druid-coordinator
     environment:
       - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
-      - druid_coordinator_centralizedTableSchema_enabled=true
+      - druid_centralizedDatasourceSchema_enabled=true
       - druid_coordinator_segmentMetadata_metadataRefreshPeriod=PT15S
     depends_on:
       - druid-overlord

integration-tests/docker/druid.sh

@@ -85,7 +85,7 @@ setupData()
   # The "query" and "security" test groups require data to be setup before running the tests.
   # In particular, they requires segments to be download from a pre-existing s3 bucket.
   # This is done by using the loadSpec put into metadatastore and s3 credientials set below.
-  if [ "$DRUID_INTEGRATION_TEST_GROUP" = "query" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-retry" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-error" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "high-availability" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "ldap-security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "upgrade" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-table-schema" ]; then
+  if [ "$DRUID_INTEGRATION_TEST_GROUP" = "query" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-retry" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-error" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "high-availability" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "ldap-security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "upgrade" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-datasource-schema" ]; then
     # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.
     find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
       && cat /test-data/${DRUID_INTEGRATION_TEST_GROUP}-sample-data.sql | mysql -u root druid \

integration-tests/script/docker_compose_args.sh

@@ -71,10 +71,10 @@ getComposeArgs()
       then
         # default + with override config + schema registry container
         echo "-f ${DOCKERDIR}/docker-compose.yml -f ${DOCKERDIR}/docker-compose.schema-registry.yml"
-  elif [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-table-schema" ]
+  elif [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-datasource-schema" ]
       then
         # cluster with overriden properties for broker and coordinator
-        echo "-f ${DOCKERDIR}/docker-compose.centralized-table-schema.yml"
+        echo "-f ${DOCKERDIR}/docker-compose.centralized-datasource-schema.yml"
   else
     # default
     echo "-f ${DOCKERDIR}/docker-compose.yml"

integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java

@@ -162,5 +162,5 @@ public class TestNGGroup
 
   public static final String HTTP_ENDPOINT = "http-endpoint";
 
-  public static final String CENTRALIZED_TABLE_SCHEMA = "centralized-table-schema";
+  public static final String CENTRALIZED_DATASOURCE_SCHEMA = "centralized-datasource-schema";
 }

integration-tests/src/test/java/org/apache/druid/tests/query/ITBroadcastJoinQueryTest.java

@@ -39,7 +39,7 @@
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
-@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA})
+@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_DATASOURCE_SCHEMA})
 @Guice(moduleFactory = DruidTestModuleFactory.class)
 public class ITBroadcastJoinQueryTest extends AbstractIndexerTest
 {

integration-tests/src/test/java/org/apache/druid/tests/query/ITJdbcQueryTest.java

@@ -47,7 +47,7 @@
 import java.util.Properties;
 import java.util.Set;
 
-@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA})
+@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_DATASOURCE_SCHEMA})
 @Guice(moduleFactory = DruidTestModuleFactory.class)
 public class ITJdbcQueryTest
 {

integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java

@@ -44,7 +44,7 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
-@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA})
+@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_DATASOURCE_SCHEMA})
 @Guice(moduleFactory = DruidTestModuleFactory.class)
 public class ITSqlCancelTest
 {

integration-tests/src/test/java/org/apache/druid/tests/query/ITSystemTableQueryTest.java

@@ -29,7 +29,7 @@
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
-@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA})
+@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_DATASOURCE_SCHEMA})
 @Guice(moduleFactory = DruidTestModuleFactory.class)
 public class ITSystemTableQueryTest
 {

integration-tests/src/test/java/org/apache/druid/tests/query/ITTwitterQueryTest.java

@@ -29,7 +29,7 @@
 import org.testng.annotations.Guice;
 import org.testng.annotations.Test;
 
-@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA})
+@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_DATASOURCE_SCHEMA})
 @Guice(moduleFactory = DruidTestModuleFactory.class)
 public class ITTwitterQueryTest
 {

integration-tests/src/test/java/org/apache/druid/tests/query/ITUnionQueryTest.java

@@ -58,7 +58,7 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA})
+@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_DATASOURCE_SCHEMA})
 @Guice(moduleFactory = DruidTestModuleFactory.class)
 public class ITUnionQueryTest extends AbstractIndexerTest
 {

integration-tests/src/test/java/org/apache/druid/tests/query/ITWikipediaQueryTest.java

@@ -47,7 +47,7 @@
 import java.util.UUID;
 import java.util.concurrent.Future;
 
-@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_TABLE_SCHEMA})
+@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_DATASOURCE_SCHEMA})
 @Guice(moduleFactory = DruidTestModuleFactory.class)
 public class ITWikipediaQueryTest
 {