remove search auto strategy, estimateSelectivity of BitmapColumnIndex

clintropolis · Dec 13, 2023 · bbbc76c · bbbc76c
1 parent 38f3cf9
commit bbbc76c
Show file tree

Hide file tree

Showing 33 changed files with 3 additions and 589 deletions.
diff --git a/...c/test/java/org/apache/druid/benchmark/DictionaryEncodedStringIndexSupplierBenchmark.java b/...c/test/java/org/apache/druid/benchmark/DictionaryEncodedStringIndexSupplierBenchmark.java
@@ -145,6 +145,5 @@ private Iterable<Integer> intGenerator()
   public void doValueSetCheck(Blackhole blackhole, BenchmarkState state)
   {
     BitmapColumnIndex bitmapIndex = state.stringValueSetIndex.forSortedValuesUtf8(state.values);
-    bitmapIndex.estimateSelectivity(10_000_000);
   }
 }
diff --git a/docs/querying/searchquery.md b/docs/querying/searchquery.md
@@ -126,10 +126,6 @@ are unique.
 queryableIndexSegment, and then evaluates search predicates. If some filters support bitmap indexes, the cursor can read
 only the rows which satisfy those filters, thereby saving I/O cost. However, it might be slow with filters of low selectivity.
 
-- "auto" strategy uses a cost-based planner for choosing an optimal search strategy. It estimates the cost of index-only
-and cursor-based execution plans, and chooses the optimal one. Currently, it is not enabled by default due to the overhead
-of cost estimation.
-
 ## Server configuration
 
 The following runtime properties apply:

diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java
@@ -70,25 +70,6 @@ default VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory
     throw new UOE("Filter[%s] cannot vectorize", getClass().getName());
   }
 
-  /**
-   * Estimate selectivity of this filter.
-   * This method can be used for cost-based query planning like in {@link org.apache.druid.query.search.AutoStrategy}.
-   * To avoid significant performance degradation for calculating the exact cost,
-   * implementation of this method targets to achieve rapid selectivity estimation
-   * with reasonable sacrifice of the accuracy.
-   * As a result, the estimated selectivity might be different from the exact value.
-   *
-   * @param indexSelector Object used to retrieve indexes
-   *
-   * @return an estimated selectivity ranging from 0 (filter selects no rows) to 1 (filter selects all rows).
-   *
-   * @see Filter#getBitmapColumnIndex(ColumnIndexSelector)
-   */
-  default double estimateSelectivity(ColumnIndexSelector indexSelector)
-  {
-    return getBitmapColumnIndex(indexSelector).estimateSelectivity(indexSelector.getNumRows());
-  }
-
   /**
    * Indicates whether this filter supports selectivity estimation.
    * A filter supports selectivity estimation if it supports bitmap index and

diff --git a/processing/src/main/java/org/apache/druid/query/search/AutoStrategy.java b/processing/src/main/java/org/apache/druid/query/search/AutoStrategy.java
diff --git a/processing/src/main/java/org/apache/druid/query/search/SearchStrategySelector.java b/processing/src/main/java/org/apache/druid/query/search/SearchStrategySelector.java
@@ -40,9 +40,8 @@ public SearchStrategy strategize(SearchQuery query)
     final String strategyString = config.withOverrides(query).getSearchStrategy();
 
     switch (strategyString) {
-      case AutoStrategy.NAME:
-        log.debug("Auto strategy is selected, query id [%s]", query.getId());
-        return AutoStrategy.of(query);
+      case "auto":
+        log.debug("Auto strategy is selected but has been removed, using 'use-index' strategy instead for query id [%s]", query.getId());
       case UseIndexesStrategy.NAME:
         log.debug("Use-index strategy is selected, query id [%s]", query.getId());
         return UseIndexesStrategy.of(query);

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/AndFilter.java
@@ -114,17 +114,6 @@ public ColumnIndexCapabilities getIndexCapabilities()
         return finalMerged;
       }
 
-      @Override
-      public double estimateSelectivity(int totalRows)
-      {
-        // Estimate selectivity with attribute value independence assumption
-        double selectivity = 1.0;
-        for (final Filter filter : filters) {
-          selectivity *= filter.estimateSelectivity(selector);
-        }
-        return selectivity;
-      }
-
       @Override
       public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
       {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java
@@ -169,15 +169,6 @@ public ColumnIndexCapabilities getIndexCapabilities()
         return rangeIndex.getIndexCapabilities().merge(nullBitmap.getIndexCapabilities());
       }
 
-      @Override
-      public double estimateSelectivity(int totalRows)
-      {
-        return Math.min(
-            1.0,
-            rangeIndex.estimateSelectivity(totalRows) + nullBitmap.estimateSelectivity(totalRows)
-        );
-      }
-
       @Override
       public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
       {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ColumnComparisonFilter.java
@@ -153,12 +153,6 @@ public Set<String> getRequiredColumns()
     return dimensions.stream().map(DimensionSpec::getDimension).collect(Collectors.toSet());
   }
 
-  @Override
-  public double estimateSelectivity(ColumnIndexSelector indexSelector)
-  {
-    throw new UnsupportedOperationException();
-  }
-
   @Override
   public boolean equals(Object o)
   {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java
@@ -240,13 +240,6 @@ public boolean supportsSelectivityEstimation(
     return false;
   }
 
-  @Override
-  public double estimateSelectivity(final ColumnIndexSelector indexSelector)
-  {
-    // Selectivity estimation not supported.
-    throw new UnsupportedOperationException();
-  }
-
   @Override
   public Set<String> getRequiredColumns()
   {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/FalseFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/FalseFilter.java
@@ -55,12 +55,6 @@ public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector)
     return new AllFalseBitmapColumnIndex(selector.getBitmapFactory());
   }
 
-  @Override
-  public double estimateSelectivity(ColumnIndexSelector indexSelector)
-  {
-    return 0;
-  }
-
   @Override
   public ValueMatcher makeMatcher(ColumnSelectorFactory factory)
   {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/Filters.java b/processing/src/main/java/org/apache/druid/segment/filter/Filters.java
@@ -52,7 +52,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Objects;
@@ -174,20 +173,6 @@ public static boolean supportsSelectivityEstimation(
     return false;
   }
 
-  public static double estimateSelectivity(
-      final Iterator<ImmutableBitmap> bitmaps,
-      final long totalNumRows
-  )
-  {
-    long numMatchedRows = 0;
-    while (bitmaps.hasNext()) {
-      final ImmutableBitmap bitmap = bitmaps.next();
-      numMatchedRows += bitmap.size();
-    }
-
-    return Math.min(1, (double) numMatchedRows / totalNumRows);
-  }
-
   @Nullable
   public static Filter convertToCNFFromQueryContext(Query query, @Nullable Filter filter)
   {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java
@@ -79,12 +79,6 @@ public ColumnIndexCapabilities getIndexCapabilities()
           return baseIndex.getIndexCapabilities();
         }
 
-        @Override
-        public double estimateSelectivity(int totalRows)
-        {
-          return 1. - baseFilter.estimateSelectivity(selector);
-        }
-
         @Override
         public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
         {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java
@@ -81,12 +81,6 @@ public ColumnIndexCapabilities getIndexCapabilities()
           return baseIndex.getIndexCapabilities();
         }
 
-        @Override
-        public double estimateSelectivity(int totalRows)
-        {
-          return 1. - baseFilter.estimateSelectivity(selector);
-        }
-
         @Override
         public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
         {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/OrFilter.java
@@ -100,17 +100,6 @@ public ColumnIndexCapabilities getIndexCapabilities()
         return finalMerged;
       }
 
-      @Override
-      public double estimateSelectivity(int totalRows)
-      {
-        // Estimate selectivity with attribute value independence assumption
-        double selectivity = 0;
-        for (final Filter filter : filters) {
-          selectivity += filter.estimateSelectivity(selector);
-        }
-        return Math.min(selectivity, 1.);
-      }
-
       @Override
       public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
       {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java
@@ -90,13 +90,6 @@ public ColumnIndexCapabilities getIndexCapabilities()
         return new SimpleColumnIndexCapabilities(true, true);
       }
 
-      @Override
-      public double estimateSelectivity(int totalRows)
-      {
-        // selectivity estimation for multi-value columns is not implemented yet.
-        throw new UnsupportedOperationException();
-      }
-
       @Override
       public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
       {
@@ -129,13 +122,6 @@ public Set<String> getRequiredColumns()
     return ImmutableSet.of(dimension);
   }
 
-  @Override
-  public double estimateSelectivity(ColumnIndexSelector indexSelector)
-  {
-    // selectivity estimation for multi-value columns is not implemented yet.
-    throw new UnsupportedOperationException();
-  }
-
   @Override
   public boolean equals(Object o)
   {

diff --git a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java
@@ -100,12 +100,6 @@ public Filter rewriteRequiredColumns(Map<String, String> columnRewrites)
     return this;
   }
 
-  @Override
-  public double estimateSelectivity(ColumnIndexSelector indexSelector)
-  {
-    return 1;
-  }
-
   @Override
   public String toString()
   {

diff --git a/processing/src/main/java/org/apache/druid/segment/index/AllFalseBitmapColumnIndex.java b/processing/src/main/java/org/apache/druid/segment/index/AllFalseBitmapColumnIndex.java
@@ -50,12 +50,6 @@ public ColumnIndexCapabilities getIndexCapabilities()
     return SimpleColumnIndexCapabilities.getConstant();
   }
 
-  @Override
-  public double estimateSelectivity(int totalRows)
-  {
-    return 0;
-  }
-
   @Override
   public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
   {