add native filters for "(filter) is true" and "(filter) is false" (#1…

…5182)

* add native filters for "(filter) is true" and "(filter) is false"

changes:
* add IsTrueDimFilter, IsFalseDimFilter, and abstract IsBooleanDimFilter for native json filter implementations of `(filter) IS TRUE` and `(filter) IS FALSE`
* add IsBooleanFilter for actual filtering logic for these filters, which ignore includeUnknown to always use matches with false for true and !matches with true for false
* fix test incorrectly adjusted to wrong answer in #15058
* add tests for default value mode

processing/src/main/java/org/apache/druid/query/filter/DimFilter.java

@@ -51,7 +51,9 @@
     @JsonSubTypes.Type(name = "false", value = FalseDimFilter.class),
     @JsonSubTypes.Type(name = "null", value = NullFilter.class),
     @JsonSubTypes.Type(name = "equals", value = EqualityFilter.class),
-    @JsonSubTypes.Type(name = "range", value = RangeFilter.class)
+    @JsonSubTypes.Type(name = "range", value = RangeFilter.class),
+    @JsonSubTypes.Type(name = "isfalse", value = IsFalseDimFilter.class),
+    @JsonSubTypes.Type(name = "istrue", value = IsTrueDimFilter.class)
 })
 public interface DimFilter extends Cacheable
 {

processing/src/main/java/org/apache/druid/query/filter/DimFilterUtils.java

@@ -58,6 +58,8 @@ public class DimFilterUtils
   static final byte EQUALS_CACHE_ID = 0x13;
   static final byte RANGE_CACHE_ID = 0x14;
 
+  static final byte IS_FILTER_BOOLEAN_FILTER_CACHE_ID = 0x15;
+
 
   public static final byte STRING_SEPARATOR = (byte) 0xFF;
 

processing/src/main/java/org/apache/druid/query/filter/IsBooleanDimFilter.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.filter;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.RangeSet;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.query.cache.CacheKeyBuilder;
+import org.apache.druid.segment.filter.IsBooleanFilter;
+
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Abstract SQL three-value logic wrapper for some child {@link DimFilter} to implement '{filter} IS TRUE' and
+ * '{filter} IS FALSE'.
+ *
+ * @see IsTrueDimFilter   - IS TRUE
+ * @see IsFalseDimFilter  - IS FALSE
+ * @see IsBooleanFilter   - actual filtering logic
+ */
+public abstract class IsBooleanDimFilter extends AbstractOptimizableDimFilter
+{
+  private final DimFilter field;
+  private final boolean isTrue;
+
+  public IsBooleanDimFilter(
+      DimFilter field,
+      boolean isTrue
+  )
+  {
+    if (field == null) {
+      throw DruidException.forPersona(DruidException.Persona.USER)
+                          .ofCategory(DruidException.Category.INVALID_INPUT)
+                          .build("IS %s operator requires a non-null filter for field", isTrue ? "TRUE" : "FALSE");
+    }
+    this.field = field;
+    this.isTrue = isTrue;
+  }
+
+  @JsonProperty("field")
+  public DimFilter getField()
+  {
+    return field;
+  }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    return new CacheKeyBuilder(DimFilterUtils.IS_FILTER_BOOLEAN_FILTER_CACHE_ID).appendBoolean(isTrue)
+                                                                                .appendCacheable(field)
+                                                                                .build();
+  }
+
+  @Override
+  public Filter toFilter()
+  {
+    return new IsBooleanFilter(field.toFilter(), isTrue);
+  }
+
+  @Override
+  public RangeSet<String> getDimensionRangeSet(String dimension)
+  {
+    return null;
+  }
+
+  @Override
+  public Set<String> getRequiredColumns()
+  {
+    return field.getRequiredColumns();
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    IsBooleanDimFilter that = (IsBooleanDimFilter) o;
+
+    if (field != null ? !field.equals(that.field) : that.field != null) {
+      return false;
+    }
+
+    return isTrue == that.isTrue;
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hash(field, isTrue);
+  }
+
+  @Override
+  public String toString()
+  {
+    return "(" + field + ") IS " + (isTrue ? "TRUE" : "FALSE");
+  }
+}

processing/src/main/java/org/apache/druid/query/filter/IsFalseDimFilter.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.filter;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class IsFalseDimFilter extends IsBooleanDimFilter
+{
+  public static IsFalseDimFilter of(DimFilter field)
+  {
+    return new IsFalseDimFilter(field);
+  }
+
+  @JsonCreator
+  public IsFalseDimFilter(
+      @JsonProperty("field") DimFilter field
+  )
+  {
+    super(field, false);
+  }
+
+  @Override
+  public DimFilter optimize()
+  {
+    return new IsFalseDimFilter(getField().optimize());
+  }
+}

processing/src/main/java/org/apache/druid/query/filter/IsTrueDimFilter.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.filter;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class IsTrueDimFilter extends IsBooleanDimFilter
+{
+  public static IsTrueDimFilter of(DimFilter field)
+  {
+    return new IsTrueDimFilter(field);
+  }
+
+  @JsonCreator
+  public IsTrueDimFilter(
+      @JsonProperty("field") DimFilter field
+  )
+  {
+    super(field, true);
+  }
+
+  @Override
+  public DimFilter optimize()
+  {
+    return new IsTrueDimFilter(getField().optimize());
+  }
+}

processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java

@@ -186,7 +186,7 @@ public boolean matches(boolean includeUnknown)
               return Arrays.stream(dResult).filter(Objects::nonNull).anyMatch(o -> Evals.asBoolean((double) o));
           }
         }
-        return (includeUnknown && eval.value() == null) || eval.asBoolean();
+        return eval.asBoolean();
       }
 
       @Override