use of org.apache.druid.segment.ColumnSelectorBitmapIndexSelector in project druid by druid-io.
the class UseIndexesStrategy method getExecutionPlan.
@Override
public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment segment) {
final ImmutableList.Builder<SearchQueryExecutor> builder = ImmutableList.builder();
final QueryableIndex index = segment.asQueryableIndex();
final StorageAdapter adapter = segment.asStorageAdapter();
final List<DimensionSpec> searchDims = getDimsToSearch(adapter.getAvailableDimensions(), query.getDimensions());
if (index != null) {
// pair of bitmap dims and non-bitmap dims
final Pair<List<DimensionSpec>, List<DimensionSpec>> pair = partitionDimensionList(adapter, searchDims);
final List<DimensionSpec> bitmapSuppDims = pair.lhs;
final List<DimensionSpec> nonBitmapSuppDims = pair.rhs;
if (bitmapSuppDims.size() > 0) {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
// from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps.
if (filter == null || filter.supportsBitmapIndex(selector)) {
final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(index, segment, filter, interval);
builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims));
} else {
// Fall back to cursor-based execution strategy
nonBitmapSuppDims.addAll(bitmapSuppDims);
}
}
if (nonBitmapSuppDims.size() > 0) {
builder.add(new CursorBasedExecutor(query, segment, filter, interval, nonBitmapSuppDims));
}
} else {
builder.add(new CursorBasedExecutor(query, segment, filter, interval, searchDims));
}
return builder.build();
}
use of org.apache.druid.segment.ColumnSelectorBitmapIndexSelector in project druid by druid-io.
the class FilterPartitionTest method testAnalyze.
@Test
public void testAnalyze() {
if (!(adapter instanceof QueryableIndexStorageAdapter)) {
return;
}
QueryableIndexStorageAdapter storageAdapter = (QueryableIndexStorageAdapter) adapter;
final ColumnSelectorBitmapIndexSelector bitmapIndexSelector = storageAdapter.makeBitmapIndexSelector(BaseFilterTest.VIRTUAL_COLUMNS);
// has bitmap index, will use it by default
Filter normalFilter = new SelectorFilter("dim1", "HELLO");
QueryableIndexStorageAdapter.FilterAnalysis filterAnalysisNormal = storageAdapter.analyzeFilter(normalFilter, bitmapIndexSelector, null);
Assert.assertTrue(filterAnalysisNormal.getPreFilterBitmap() != null);
Assert.assertTrue(filterAnalysisNormal.getPostFilter() == null);
// no bitmap index, should be a post filter
Filter noBitmapFilter = new NoBitmapSelectorFilter("dim1", "HELLO");
QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterAnalysis = storageAdapter.analyzeFilter(noBitmapFilter, bitmapIndexSelector, null);
Assert.assertTrue(noBitmapFilterAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(noBitmapFilterAnalysis.getPostFilter() != null);
// this column has a bitmap index, but is forced to not use it
Filter bitmapFilterWithForceNoIndexTuning = new SelectorFilter("dim1", "HELLO", new FilterTuning(false, null, null));
QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithForceNoIndexTuningAnalysis = storageAdapter.analyzeFilter(bitmapFilterWithForceNoIndexTuning, bitmapIndexSelector, null);
Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(bitmapFilterWithForceNoIndexTuningAnalysis.getPostFilter() != null);
// this max cardinality is too low to use bitmap index
Filter bitmapFilterWithCardinalityMax = new SelectorFilter("dim1", "HELLO", new FilterTuning(true, 0, 3));
QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMaxAnalysis = storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax, bitmapIndexSelector, null);
Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(bitmapFilterWithCardinalityMaxAnalysis.getPostFilter() != null);
// this max cardinality is high enough that we can still use bitmap index
Filter bitmapFilterWithCardinalityMax2 = new SelectorFilter("dim1", "HELLO", new FilterTuning(true, 0, 1000));
QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMax2Analysis = storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMax2, bitmapIndexSelector, null);
Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPreFilterBitmap() != null);
Assert.assertTrue(bitmapFilterWithCardinalityMax2Analysis.getPostFilter() == null);
// this min cardinality is too high, will not use bitmap index
Filter bitmapFilterWithCardinalityMin = new SelectorFilter("dim1", "HELLO", new FilterTuning(true, 1000, null));
QueryableIndexStorageAdapter.FilterAnalysis bitmapFilterWithCardinalityMinAnalysis = storageAdapter.analyzeFilter(bitmapFilterWithCardinalityMin, bitmapIndexSelector, null);
Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(bitmapFilterWithCardinalityMinAnalysis.getPostFilter() != null);
// cannot force using bitmap if there are no bitmaps
Filter noBitmapFilterWithForceUse = new NoBitmapSelectorFilter("dim1", "HELLO", new FilterTuning(true, null, null));
QueryableIndexStorageAdapter.FilterAnalysis noBitmapFilterWithForceUseAnalysis = storageAdapter.analyzeFilter(noBitmapFilterWithForceUse, bitmapIndexSelector, null);
Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPreFilterBitmap() == null);
Assert.assertTrue(noBitmapFilterWithForceUseAnalysis.getPostFilter() != null);
}
use of org.apache.druid.segment.ColumnSelectorBitmapIndexSelector in project druid by druid-io.
the class ListFilteredVirtualColumnSelectorTest method testFilterListFilteredVirtualColumnDenyIndex.
@Test
public void testFilterListFilteredVirtualColumnDenyIndex() {
ListFilteredVirtualColumn virtualColumn = new ListFilteredVirtualColumn(DENY_VIRTUAL_NAME, new DefaultDimensionSpec(COLUMN_NAME, COLUMN_NAME, ColumnType.STRING), ImmutableSet.of("a", "b"), false);
ColumnSelector selector = EasyMock.createMock(ColumnSelector.class);
ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
BitmapIndex index = EasyMock.createMock(BitmapIndex.class);
ImmutableBitmap bitmap = EasyMock.createMock(ImmutableBitmap.class);
BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
EasyMock.expect(index.getCardinality()).andReturn(3).atLeastOnce();
EasyMock.expect(index.getValue(0)).andReturn("a").atLeastOnce();
EasyMock.expect(index.getValue(1)).andReturn("b").atLeastOnce();
EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
EasyMock.expect(index.getBitmap(0)).andReturn(bitmap).once();
EasyMock.expect(index.getBitmapFactory()).andReturn(bitmapFactory).once();
EasyMock.expect(index.hasNulls()).andReturn(true).once();
EasyMock.replay(selector, holder, index, bitmap, bitmapFactory);
ColumnSelectorBitmapIndexSelector bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(new RoaringBitmapFactory(), VirtualColumns.create(Collections.singletonList(virtualColumn)), selector);
SelectorFilter filter = new SelectorFilter(DENY_VIRTUAL_NAME, "c");
Assert.assertTrue(filter.shouldUseBitmapIndex(bitmapIndexSelector));
BitmapIndex listFilteredIndex = bitmapIndexSelector.getBitmapIndex(DENY_VIRTUAL_NAME);
Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
Assert.assertEquals(-1, listFilteredIndex.getIndex("b"));
Assert.assertEquals(0, listFilteredIndex.getIndex("c"));
Assert.assertEquals(1, listFilteredIndex.getCardinality());
Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
Assert.assertEquals(bitmapFactory, listFilteredIndex.getBitmapFactory());
Assert.assertTrue(listFilteredIndex.hasNulls());
EasyMock.verify(selector, holder, index, bitmap, bitmapFactory);
}
use of org.apache.druid.segment.ColumnSelectorBitmapIndexSelector in project druid by druid-io.
the class AutoStrategy method getExecutionPlan.
@Override
public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment segment) {
final QueryableIndex index = segment.asQueryableIndex();
if (index != null) {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
// from the non-bitmap-support filters, and then use it to compute the filtered result by intersecting bitmaps.
if (filter == null || filter.supportsSelectivityEstimation(index, selector)) {
final List<DimensionSpec> dimsToSearch = getDimsToSearch(index.getAvailableDimensions(), query.getDimensions());
// Choose a search query execution strategy depending on the query.
// The costs of index-only plan and cursor-based plan can be computed like below.
//
// c_index = (total cardinality of all search dimensions) * (bitmap intersection cost)
// * (search predicate processing cost)
// c_cursor = (# of rows in a segment) * (filter selectivity) * (# of dimensions)
// * (search predicate processing cost)
final SearchQueryDecisionHelper helper = getDecisionHelper(index);
final double useIndexStrategyCost = helper.getBitmapIntersectCost() * computeTotalCard(index, dimsToSearch);
final double cursorOnlyStrategyCost = (filter == null ? 1. : filter.estimateSelectivity(selector)) * selector.getNumRows() * dimsToSearch.size();
log.debug("Use-index strategy cost: %f, cursor-only strategy cost: %f", useIndexStrategyCost, cursorOnlyStrategyCost);
if (useIndexStrategyCost < cursorOnlyStrategyCost) {
log.debug("Use-index execution strategy is selected, query id [%s]", query.getId());
return UseIndexesStrategy.of(query).getExecutionPlan(query, segment);
} else {
log.debug("Cursor-only execution strategy is selected, query id [%s]", query.getId());
return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
}
} else {
log.debug("Filter doesn't support bitmap index. Fall back to cursor-only execution strategy, query id [%s]", query.getId());
return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
}
} else {
log.debug("Index doesn't exist. Fall back to cursor-only execution strategy, query id [%s]", query.getId());
return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
}
}
use of org.apache.druid.segment.ColumnSelectorBitmapIndexSelector in project druid by druid-io.
the class UseIndexesStrategy method makeTimeFilteredBitmap.
static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, final Segment segment, final Filter filter, final Interval interval) {
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
final ImmutableBitmap baseFilter;
if (filter == null) {
baseFilter = null;
} else {
final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter);
baseFilter = filter.getBitmapIndex(selector);
}
final ImmutableBitmap timeFilteredBitmap;
if (!interval.contains(segment.getDataInterval())) {
final MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap();
final ColumnHolder timeColumnHolder = index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME);
try (final NumericColumn timeValues = (NumericColumn) timeColumnHolder.getColumn()) {
int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true));
int endIndex = Math.min(timeValues.length() - 1, getStartIndexOfTime(timeValues, interval.getEndMillis(), false));
for (int i = startIndex; i <= endIndex; i++) {
timeBitmap.add(i);
}
final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap);
timeFilteredBitmap = (baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter);
}
} else {
timeFilteredBitmap = baseFilter;
}
return timeFilteredBitmap;
}
Aggregations