#7714 ignore query options from commented out queries

.github/workflows/pinot_tests.yml

@@ -38,6 +38,19 @@ on:
       - "**.md"
 
 jobs:
+  linter-test:
+    runs-on: ubuntu-latest
+    name: Pinot Linter Test Set
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Linter Test
+        env:
+          MAVEN_OPTS: -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+        run: .github/workflows/scripts/.pinot_linter.sh
   unit-test:
     runs-on: ubuntu-latest
     strategy:
@@ -101,7 +114,7 @@ jobs:
     strategy:
       matrix:
         test_suite: [ "compatibility-verifier/sample-test-suite" ]
-        old_commit: [ "release-0.7.1", "master" ]
+        old_commit: [ "release-0.9.0", "master" ]
     name: Pinot Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
     steps:
       - uses: actions/checkout@v2

.github/workflows/scripts/.pinot_linter.sh

@@ -0,0 +1,33 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java version
+java -version
+
+# Check network
+ifconfig
+netstat -i
+
+
+mvn license:check || exit 1
+mvn checkstyle:check || exit 1
+mvn spotless:check || exit 1
+mvn enforcer:enforce || exit 1
+

.github/workflows/scripts/.pinot_quickstart.sh

@@ -85,7 +85,8 @@ fi
 DIST_BIN_DIR=`ls -d pinot-distribution/target/apache-pinot-*/apache-pinot-*`
 cd "${DIST_BIN_DIR}"
 
-# Test standalone pinot
+# Test standalone pinot. Configure JAVA_OPTS for smaller memory, and don't use System.exit
+export JAVA_OPTS="-Xms1G -Dlog4j2.configurationFile=conf/log4j2.xml"
 bin/pinot-admin.sh StartZookeeper &
 ZK_PID=$!
 sleep 10

LICENSE-binary

@@ -205,6 +205,7 @@ This project bundles some components that are also licensed under the Apache
 License Version 2.0:
 
 com.101tec:zkclient:0.7
+com.chuusai:shapeless_2.12:2.3.3
 com.clearspring.analytics:stream:2.7.0
 com.fasterxml.jackson.core:jackson-annotations:2.10.0
 com.fasterxml.jackson.core:jackson-core:2.10.0
@@ -217,6 +218,8 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.10.0
 com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.10.0
 com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.10.0
 com.fasterxml.jackson.module:jackson-module-jsonSchema:2.11.1
+com.fasterxml.jackson.module:jackson-module-paranamer:2.10.0
+com.fasterxml.jackson.module:jackson-module-scala_2.12:2.10.0
 com.fasterxml.woodstox:woodstox-core:6.0.1
 com.github.ben-manes.caffeine:caffeine:2.6.2
 com.google.android:annotations:4.1.1.4
@@ -260,12 +263,19 @@ commons-io:commons-io:2.4
 commons-lang:commons-lang:2.6
 commons-logging:commons-logging:1.2
 commons-pool:commons-pool:1.6
+info.picocli:picocli:4.6.1
 io.airlift:aircompressor:0.10
+io.circe:circe-core_2.12:0.13.0
+io.circe:circe-generic_2.12:0.13.0
+io.circe:circe-jawn_2.12:0.13.0
+io.circe:circe-numbers_2.12:0.13.0
+io.circe:circe-parser_2.12:0.13.0
 io.confluent:common-config:5.3.1
 io.confluent:common-utils:5.3.1
 io.confluent:kafka-avro-serializer:5.3.1
 io.confluent:kafka-schema-registry-client:5.3.1
-io.dropwizard.metrics:metrics-core:4.1.2
+io.dropwizard.metrics:metrics-core:4.2.2
+io.dropwizard.metrics:metrics-jmx:4.2.2
 io.grpc:grpc-api:1.30.0
 io.grpc:grpc-context:1.14.0
 io.grpc:grpc-core:1.30.0
@@ -309,7 +319,6 @@ it.unimi.dsi:fastutil:8.2.3
 javax.inject:javax.inject:1
 javax.validation:validation-api:2.0.1.Final
 joda-time:joda-time:2.10.5
-me.lemire.integercompression:JavaFastPFOR:0.0.13
 net.minidev:accessors-smart:1.2
 net.minidev:json-smart:2.3
 org.apache.avro:avro-protobuf:1.9.1
@@ -336,10 +345,10 @@ org.apache.httpcomponents:httpmime:4.5.3
 org.apache.kafka:kafka-clients:2.0.0
 org.apache.kafka:kafka_2.10:0.9.0.1
 org.apache.kafka:kafka_2.11:2.0.0
-org.apache.logging.log4j:log4j-1.2-api:2.11.2
-org.apache.logging.log4j:log4j-api:2.11.2
-org.apache.logging.log4j:log4j-core:2.11.2
-org.apache.logging.log4j:log4j-slf4j-impl:2.11.2
+org.apache.logging.log4j:log4j-1.2-api:2.15.0
+org.apache.logging.log4j:log4j-api:2.15.0
+org.apache.logging.log4j:log4j-core:2.15.0
+org.apache.logging.log4j:log4j-slf4j-impl:2.15.0
 org.apache.lucene:lucene-analyzers-common:8.2.0
 org.apache.lucene:lucene-core:8.2.0
 org.apache.lucene:lucene-queries:8.2.0
@@ -380,8 +389,9 @@ org.eclipse.jetty:jetty-util:9.4.39.v20210325
 org.javassist:javassist:3.19.0-GA
 org.lz4:lz4-java:1.7.1
 org.quartz-scheduler:quartz:2.3.2
-org.roaringbitmap:RoaringBitmap:0.9.0
-org.roaringbitmap:shims:0.9.0
+org.roaringbitmap:RoaringBitmap:0.9.23
+org.roaringbitmap:shims:0.9.23
+org.typelevel:macro-compat_2.12:1.1.1
 org.webjars:swagger-ui:3.18.2
 org.wildfly.openssl:wildfly-openssl:1.0.7.Final
 org.xerial.larray:larray-buffer:0.4.1
@@ -406,6 +416,7 @@ software.amazon.awssdk:protocol-core:2.14.28
 software.amazon.awssdk:regions:2.14.28
 software.amazon.awssdk:s3:2.14.28
 software.amazon.awssdk:sdk-core:2.14.28
+software.amazon.awssdk:sts:2.14.28
 software.amazon.awssdk:utils:2.14.28
 software.amazon.eventstream:eventstream:1.0.1
 
@@ -418,7 +429,6 @@ of these licenses.
 
 MIT License
 -----------
-args4j:args4j:2.32
 com.azure:azure-core-http-netty:1.7.1
 com.azure:azure-core:1.12.0
 com.azure:azure-storage-blob:12.10.0
@@ -429,10 +439,19 @@ com.microsoft.azure:azure-data-lake-store-sdk:2.3.9
 net.sf.jopt-simple:jopt-simple:4.6
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.reactivestreams:reactive-streams:1.0.3
+org.typelevel:cats-core_2.12:2.1.0
+org.typelevel:cats-kernel_2.12:2.1.0
+org.typelevel:cats-macros_2.12:2.1.0
+org.typelevel:jawn-parser_2.12:1.0.0
 org.slf4j:jul-to-slf4j:1.7.25
 org.slf4j:slf4j-api:1.7.25
 
 
+BSD
+---
+com.thoughtworks.paranamer:paranamer:2.8
+
+
 BSD 2-Clause
 ------------
 com.github.luben:zstd-jni:1.4.9-5
@@ -454,6 +473,7 @@ com.jcabi:jcabi-log:0.17.1
 org.antlr:antlr4-runtime:4.6
 org.ow2.asm:asm:5.0.4
 org.scala-lang:scala-library:2.11.11
+org.scala-lang.modules:scala-xml_2.12:1.3.0
 org.threeten:threeten-extra:1.5.0
 org.threeten:threetenbp:1.3.3
 
@@ -527,7 +547,7 @@ org.jvnet.mimepull:mimepull:1.9.11
 
 WTFPL License
 -------------
-org.reflections:reflections:0.9.11
+org.reflections:reflections:0.9.9
 
 
 Creative Commons Attribution License (CC BY 2.5)
@@ -540,3 +560,8 @@ Bounty Castle License
 org.bouncycastle:bcpkix-jdk15to18:1.68
 org.bouncycastle:bcprov-ext-jdk15to18:1.68
 org.bouncycastle:bcprov-jdk15to18:1.68
+
+
+GNU Lesser Public License
+-------------------------
+com.google.code.findbugs:annotations:jar:2.0.1

NOTICE-binary

@@ -19,23 +19,29 @@ Apache Commons Configuration
 Copyright 2001-2013 The Apache Software Foundation
 
 Apache Commons Codec
-Copyright 2002-2011 The Apache Software Foundation
+Copyright 2002-2020 The Apache Software Foundation
 
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
+This product includes software developed at
+The Apache Software Foundation (https://www.apache.org/).
 
---------------------------------------------------------------------------------
-src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java contains
-test data from http://aspell.sourceforge.net/test/batch0.tab.
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson ([email protected])
 
-Copyright (C) 2002 Kevin Atkinson ([email protected]). Verbatim copying
-and distribution of this entire article is permitted in any medium,
-provided this notice is preserved.
---------------------------------------------------------------------------------
+===============================================================================
+
+The content of package org.apache.commons.codec.language.bm has been translated
+from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
 
 Apache Commons IO
 Copyright 2002-2012 The Apache Software Foundation
 
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).
+
 Apache Commons Lang
 Copyright 2001-2011 The Apache Software Foundation
 
@@ -131,9 +137,6 @@ Copyright 1999-2007 The Apache Software Foundation
 Apache Commons Compress
 Copyright 2002-2020 The Apache Software Foundation
 
-This product includes software developed at
-The Apache Software Foundation (https://www.apache.org/).
-
 ---
 
 The files in the package org.apache.commons.compress.archivers.sevenz
@@ -599,8 +602,6 @@ These files are used as test data and test result specifications.
 Apache Commons Math
 Copyright 2001-2013 The Apache Software Foundation
 
-===============================================================================
-
 The inverse error function implementation in the Erf class is based on CUDA
 code developed by Mike Giles, Oxford-Man Institute of Quantitative Finance,
 and published in GPU Computing Gems, volume 2, 2010.
@@ -695,3 +696,4 @@ Copyright 2001-2012 The Apache Software Foundation
 
 ResolverUtil.java
 Copyright 2005-2006 Tim Fennell
+

README.md

@@ -87,7 +87,7 @@ SELECT sum(clicks), sum(impressions) FROM AdAnalyticsTable
        daysSinceEpoch TOP 100
 ```
 
-Pinot is not a replacement for database i.e it cannot be used as source of truth store, cannot mutate data. While Pinot [supports text search](https://docs.pinot.apache.org/basics/features/text-search-support), it's not a replacement for a search engine. Also, Pinot queries cannot span across multiple tables by default. You can use the [Presto-Pinot connector](https://prestodb.io/docs/current/connector/pinot.html) to achieve table joins and other features.
+Pinot is not a replacement for database i.e it cannot be used as source of truth store, cannot mutate data. While Pinot [supports text search](https://docs.pinot.apache.org/basics/features/text-search-support), it's not a replacement for a search engine. Also, Pinot queries cannot span across multiple tables by default. You can use the [Trino-Pinot Connector](https://trino.io/docs/current/connector/pinot.html) or [Presto-Pinot Connector](https://prestodb.io/docs/current/connector/pinot.html) to achieve table joins and other features.
 
 ## Building Pinot
 More detailed instructions can be found at [Quick Demo](https://docs.pinot.apache.org/getting-started) section in the documentation.
@@ -105,7 +105,7 @@ $ bin/quick-start-batch.sh
 ```
 
 ## Deploying Pinot to Kubernetes
-Please refer to [Running Pinot on Kubernetes](https://docs.pinot.apache.org/basics/getting-started/kubernetes-quickstart) in our project documentation. Pinot also provides Kubernetes integrations with the interactive query engine, [Presto](kubernetes/helm/presto-coordinator.yaml), and the data visualization tool, [Apache Superset](kubernetes/helm/superset.yaml).
+Please refer to [Running Pinot on Kubernetes](https://docs.pinot.apache.org/basics/getting-started/kubernetes-quickstart) in our project documentation. Pinot also provides Kubernetes integrations with the interactive query engine, [Trino](https://docs.pinot.apache.org/integrations/trino) [Presto](https://docs.pinot.apache.org/integrations/presto), and the data visualization tool, [Apache Superset](kubernetes/helm/superset.yaml).
 
 ## Join the Community
  - Ask questions on [Apache Pinot Slack](https://join.slack.com/t/apache-pinot/shared_invite/zt-5z7pav2f-yYtjZdVA~EDmrGkho87Vzw)

compatibility-verifier/compCheck.sh

@@ -230,7 +230,20 @@ function setupCompatTester() {
     cd ${pinotIntegTestsRelDir}
     pwd
   ))
-  CLASSPATH_PREFIX=$(ls ${pinotIntegTestsAbsDir}/pinot-integration-tests-*-tests.jar)
+  JAR_LIST=$(ls ${pinotIntegTestsAbsDir}/pinot-integration-tests-*-tests.jar)
+  CLASSPATH_PREFIX="$(echo $JAR_LIST | tr ' ' :)"
+  # Adding pinot-integration-test-base JAR
+  # TODO remove this condition once released.
+  local pinotIntegTestBaseRelDir="$(dirname $0)/../pinot-integration-test-base/target"
+  if [[ -d "$pinotIntegTestBaseRelDir" ]]; then
+    local pinotIntegTestBaseAbsDir=$( (
+      cd ${pinotIntegTestBaseRelDir}
+      pwd
+    ))
+    JAR_LIST="$(ls ${pinotIntegTestBaseAbsDir}/pinot-integration-test-base-*.jar)"
+    CLASSPATH_PREFIX="$CLASSPATH_PREFIX:$(echo $JAR_LIST | tr ' ' :)"
+  fi
+  echo "CLASSPATH_PREFIX is set as: $CLASSPATH_PREFIX"
   export CLASSPATH_PREFIX
 }
 

config/checkstyle.xml

@@ -112,6 +112,55 @@
     <module name="AvoidStarImport">
       <property name="allowStaticMemberImports" value="true"/>
     </module>
+    <!-- no static imports except in tests -->
+    <module name="AvoidStaticImport">
+      <property name="excludes"
+                value="com.google.common.base.Preconditions.*,
+                java.lang.Math.*,
+                java.lang.Character.*,
+                java.lang.invoke.MethodHandles.*,
+                java.lang.invoke.MethodType.*,
+                java.nio.charset.StandardCharsets.*,
+                java.util.Locale.*,
+                java.util.Objects.*,
+                java.util.concurrent.TimeUnit.*,
+                org.apache.pinot.common.request.context.FilterContext.Type.*,
+                org.apache.pinot.common.request.context.predicate.Predicate.Type.*,
+                org.apache.pinot.common.segment.generation.SegmentGenerationUtils.*,
+                org.apache.pinot.controller.recommender.realtime.provisioning.MemoryEstimator.*,
+                org.apache.pinot.controller.recommender.rules.io.params.RecommenderConstants.*,
+                org.apache.pinot.controller.recommender.rules.io.params.RecommenderConstants.FlagQueryRuleParams.*,
+                org.apache.pinot.controller.recommender.rules.io.params.RecommenderConstants.InvertedSortedIndexJointRule.*,
+                org.apache.pinot.controller.recommender.rules.io.params.RecommenderConstants.NoDictionaryOnHeapDictionaryJointRule.*,
+                org.apache.pinot.controller.recommender.rules.io.params.RecommenderConstants.PartitionRule.*,
+                org.apache.pinot.controller.recommender.rules.io.params.RecommenderConstants.RulesToExecute.*,
+                org.apache.pinot.controller.recommender.rules.utils.PredicateParseResult.*,
+                org.apache.pinot.client.utils.Constants.*,
+                org.apache.pinot.segment.local.startree.StarTreeBuilderUtils.*,
+                org.apache.pinot.segment.local.startree.v2.store.StarTreeIndexMapUtils.*,
+                org.apache.pinot.segment.local.utils.GeometryType.*,
+                org.apache.pinot.segment.spi.Constants.*,
+                org.apache.pinot.segment.spi.V1Constants.Indexes.*,
+                org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Column.*,
+                org.apache.pinot.segment.spi.V1Constants.MetadataKeys.Segment.*,
+                org.apache.pinot.segment.spi.creator.SegmentVersion.*,
+                org.apache.pinot.segment.spi.index.startree.StarTreeV2Constants.*,
+                org.apache.pinot.spi.data.FieldSpec.*,
+                org.apache.pinot.spi.data.FieldSpec.DataType,
+                org.apache.pinot.spi.data.FieldSpec.DataType.*,
+                org.apache.pinot.spi.data.FieldSpec.FieldType.*,
+                org.apache.pinot.spi.plugin.PluginManager.*,
+                org.apache.pinot.spi.utils.CommonConstants.*,
+                org.apache.pinot.spi.utils.CommonConstants.Broker.*,
+                org.apache.pinot.spi.utils.CommonConstants.Broker.Request.*,
+                org.apache.pinot.spi.utils.CommonConstants.Controller.*,
+                org.apache.pinot.spi.utils.CommonConstants.Helix.*,
+                org.apache.pinot.spi.utils.CommonConstants.Segment.*,
+                org.apache.pinot.spi.utils.CommonConstants.Server.*,
+                org.apache.pinot.spi.utils.CommonConstants.Server.SegmentCompletionProtocol.*,
+                org.apache.pinot.spi.utils.EqualityUtils.*,
+                org.apache.pinot.tools.Quickstart.*"/>
+    </module>
     <!-- Do not import 'sun' packages -->
     <module name="IllegalImport"/>
     <!-- Do not duplicate import statements -->
@@ -177,6 +226,7 @@
     </module>
     <module name="WhitespaceAfter"/>
     <module name="WhitespaceAround"/>
+    <module name="SingleSpaceSeparator"/>
 
     <!-- Allow Checkstyle warnings to be suppressed using trailing comments -->
     <module name="SuppressWithNearbyCommentFilter"/>
@@ -202,6 +252,10 @@
     <property name="format" value="\*[\s]*@(params|throw|returns)[\s]+"/>
     <property name="message" value="Correct misspelled Javadoc tag"/>
   </module>
+  <module name="RegexpSingleline">
+    <property name="format" value="this\._"/>
+    <property name="message" value="Do not qualify member variables with this"/>
+  </module>
 
   <!-- Read checker suppressions from a file -->
   <module name="SuppressionFilter">