Examples with FilterTuning org.apache.druid.query.filter.FilterTuning used on opensource projects

Example 1 with FilterTuning

use of org.apache.druid.query.filter.FilterTuning in project druid by druid-io.

the class FilterPartitionTest method testAnalyze.

@Test
public void testAnalyze() {
    if (!(adapter instanceof QueryableIndexStorageAdapter)) {
        return;
    }
    QueryableIndexStorageAdapter storageAdapter = (QueryableIndexStorageAdapter) adapter;
    final ColumnSelectorBitmapIndexSelector bitmapIndexSelector = storageAdapter.makeBitmapIndexSelector(BaseFilterTest.VIRTUAL_COLUMNS);
    // has bitmap index, will use it by default
    Filter normalFilter = new SelectorFilter("dim1", "HELLO");
    QueryableIndexStorageAdapter.FilterAnalysis filterAnalysisNormal = storageAdapter.analyzeFilter(normalFilter, bitmapIndexSelector, null);
    Assert.assertTrue(filterAnalysisNormal.getPreFilterBitmap() != null);
    Assert.assertTrue(filterAnalysisNormal.getPostFilter() == null);
    // no bitmap index, should be a post filter
    Filter noBitmapFilter = new NoBitmapSelectorFilter("dim1", "HELLO");
    QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterAnalysis = storageAdapter.analyzeFilter(noBitmapFilter, bitmapIndexSelector, null);
    Assert.assertTrue(noBitmapFilterAnalysis.getPreFilterBitmap() == null);
    Assert.assertTrue(noBitmapFilterAnalysis.getPostFilter() != null);
    // this column has a bitmap index, but is forced to not use it
    Filter bitmapFilterWithForceNoIndexTuning = new SelectorFilter("dim1", "HELLO", new FilterTuning(false, null, null));
    QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithForceNoIndexTuningAnalysis = storageAdapter.analyzeFilter(bitmapFilterWithForceNoIndexTuning, bitmapIndexSelector, null);
    Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPreFilterBitmap() == null);
    Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPostFilter() != null);
    // this max cardinality is too low to use bitmap index
    Filter bitmapFilterWithCardinalityMax = new SelectorFilter("dim1", "HELLO", new FilterTuning(true, 0, 3));
    QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMaxAnalysis = storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax, bitmapIndexSelector, null);
    Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPreFilterBitmap() == null);
    Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPostFilter() != null);
    // this max cardinality is high enough that we can still use bitmap index
    Filter bitmapFilterWithCardinalityMax2 = new SelectorFilter("dim1", "HELLO", new FilterTuning(true, 0, 1000));
    QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMax2Analysis = storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax2, bitmapIndexSelector, null);
    Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPreFilterBitmap() != null);
    Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPostFilter() == null);
    // this min cardinality is too high, will not use bitmap index
    Filter bitmapFilterWithCardinalityMin = new SelectorFilter("dim1", "HELLO", new FilterTuning(true, 1000, null));
    QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMinAnalysis = storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMin, bitmapIndexSelector, null);
    Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPreFilterBitmap() == null);
    Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPostFilter() != null);
    // cannot force using bitmap if there are no bitmaps
    Filter noBitmapFilterWithForceUse = new NoBitmapSelectorFilter("dim1", "HELLO", new FilterTuning(true, null, null));
    QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterWithForceUseAnalysis = storageAdapter.analyzeFilter(noBitmapFilterWithForceUse, bitmapIndexSelector, null);
    Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPreFilterBitmap() == null);
    Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPostFilter() != null);
}

Example 2 with FilterTuning

use of org.apache.druid.query.filter.FilterTuning in project druid by druid-io.

the class Filters method shouldUseBitmapIndex.

/**
 * This method provides a "standard" implementation of {@link Filter#shouldUseBitmapIndex(BitmapIndexSelector)} which takes
 * a {@link Filter}, a {@link BitmapIndexSelector}, and {@link FilterTuning} to determine if:
 * a) the filter supports bitmap indexes for all required columns
 * b) the filter tuning specifies that it should use the index
 * c) the cardinality of the column is above the minimum threshold and below the maximum threshold to use the index
 *
 * If all these things are true, {@link org.apache.druid.segment.QueryableIndexStorageAdapter} will utilize the
 * indexes.
 */
public static boolean shouldUseBitmapIndex(Filter filter, BitmapIndexSelector indexSelector, @Nullable FilterTuning filterTuning) {
    final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector);
    if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseBitmapIndex()) {
        return filter.getRequiredColumns().stream().allMatch(column -> {
            final BitmapIndex index = indexSelector.getBitmapIndex(column);
            Preconditions.checkNotNull(index, "Column does not have a bitmap index");
            final int cardinality = index.getCardinality();
            return cardinality >= tuning.getMinCardinalityToUseBitmapIndex() && cardinality <= tuning.getMaxCardinalityToUseBitmapIndex();
        });
    }
    return false;
}