Examples with BitmapIndexSelector org.apache.druid.query.filter.BitmapIndexSelector used on opensource projects

Example 1 with BitmapIndexSelector

use of org.apache.druid.query.filter.BitmapIndexSelector in project druid by druid-io.

the class UseIndexesStrategy method getExecutionPlan.

@Override
public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment segment) {
    final ImmutableList.Builder<SearchQueryExecutor> builder = ImmutableList.builder();
    final QueryableIndex index = segment.asQueryableIndex();
    final StorageAdapter adapter = segment.asStorageAdapter();
    final List<DimensionSpec> searchDims = getDimsToSearch(adapter.getAvailableDimensions(), query.getDimensions());
    if (index != null) {
        // pair of bitmap dims and non-bitmap dims
        final Pair<List<DimensionSpec>, List<DimensionSpec>> pair = partitionDimensionList(adapter, searchDims);
        final List<DimensionSpec> bitmapSuppDims = pair.lhs;
        final List<DimensionSpec> nonBitmapSuppDims = pair.rhs;
        if (bitmapSuppDims.size() > 0) {
            final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
            // from the non-bitmap-support filter, and then use it to compute the filtered result by intersecting bitmaps.
            if (filter == null || filter.supportsBitmapIndex(selector)) {
                final ImmutableBitmap timeFilteredBitmap = makeTimeFilteredBitmap(index, segment, filter, interval);
                builder.add(new IndexOnlyExecutor(query, segment, timeFilteredBitmap, bitmapSuppDims));
            } else {
                // Fall back to cursor-based execution strategy
                nonBitmapSuppDims.addAll(bitmapSuppDims);
            }
        }
        if (nonBitmapSuppDims.size() > 0) {
            builder.add(new CursorBasedExecutor(query, segment, filter, interval, nonBitmapSuppDims));
        }
    } else {
        builder.add(new CursorBasedExecutor(query, segment, filter, interval, searchDims));
    }
    return builder.build();
}

Example 2 with BitmapIndexSelector

use of org.apache.druid.query.filter.BitmapIndexSelector in project druid by druid-io.

the class AutoStrategy method getExecutionPlan.

@Override
public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment segment) {
    final QueryableIndex index = segment.asQueryableIndex();
    if (index != null) {
        final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
        // from the non-bitmap-support filters, and then use it to compute the filtered result by intersecting bitmaps.
        if (filter == null || filter.supportsSelectivityEstimation(index, selector)) {
            final List<DimensionSpec> dimsToSearch = getDimsToSearch(index.getAvailableDimensions(), query.getDimensions());
            // Choose a search query execution strategy depending on the query.
            // The costs of index-only plan and cursor-based plan can be computed like below.
            // 
            // c_index = (total cardinality of all search dimensions) * (bitmap intersection cost)
            // * (search predicate processing cost)
            // c_cursor = (# of rows in a segment) * (filter selectivity) * (# of dimensions)
            // * (search predicate processing cost)
            final SearchQueryDecisionHelper helper = getDecisionHelper(index);
            final double useIndexStrategyCost = helper.getBitmapIntersectCost() * computeTotalCard(index, dimsToSearch);
            final double cursorOnlyStrategyCost = (filter == null ? 1. : filter.estimateSelectivity(selector)) * selector.getNumRows() * dimsToSearch.size();
            log.debug("Use-index strategy cost: %f, cursor-only strategy cost: %f", useIndexStrategyCost, cursorOnlyStrategyCost);
            if (useIndexStrategyCost < cursorOnlyStrategyCost) {
                log.debug("Use-index execution strategy is selected, query id [%s]", query.getId());
                return UseIndexesStrategy.of(query).getExecutionPlan(query, segment);
            } else {
                log.debug("Cursor-only execution strategy is selected, query id [%s]", query.getId());
                return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
            }
        } else {
            log.debug("Filter doesn't support bitmap index. Fall back to cursor-only execution strategy, query id [%s]", query.getId());
            return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
        }
    } else {
        log.debug("Index doesn't exist. Fall back to cursor-only execution strategy, query id [%s]", query.getId());
        return CursorOnlyStrategy.of(query).getExecutionPlan(query, segment);
    }
}

Example 3 with BitmapIndexSelector

use of org.apache.druid.query.filter.BitmapIndexSelector in project druid by druid-io.

the class UseIndexesStrategy method makeTimeFilteredBitmap.

static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, final Segment segment, final Filter filter, final Interval interval) {
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final ImmutableBitmap baseFilter;
    if (filter == null) {
        baseFilter = null;
    } else {
        final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
        Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter);
        baseFilter = filter.getBitmapIndex(selector);
    }
    final ImmutableBitmap timeFilteredBitmap;
    if (!interval.contains(segment.getDataInterval())) {
        final MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap();
        final ColumnHolder timeColumnHolder = index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME);
        try (final NumericColumn timeValues = (NumericColumn) timeColumnHolder.getColumn()) {
            int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true));
            int endIndex = Math.min(timeValues.length() - 1, getStartIndexOfTime(timeValues, interval.getEndMillis(), false));
            for (int i = startIndex; i <= endIndex; i++) {
                timeBitmap.add(i);
            }
            final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap);
            timeFilteredBitmap = (baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter);
        }
    } else {
        timeFilteredBitmap = baseFilter;
    }
    return timeFilteredBitmap;
}

Example 4 with BitmapIndexSelector

use of org.apache.druid.query.filter.BitmapIndexSelector in project druid by druid-io.

the class BaseFilterTest method selectColumnValuesMatchingFilterUsingVectorizedPostFiltering.

private List<String> selectColumnValuesMatchingFilterUsingVectorizedPostFiltering(final DimFilter filter, final String selectColumn) {
    final Filter theFilter = makeFilter(filter);
    final Filter postFilteringFilter = new Filter() {

        @Override
        public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory) {
            throw new UnsupportedOperationException();
        }

        @Override
        public ValueMatcher makeMatcher(ColumnSelectorFactory factory) {
            return theFilter.makeMatcher(factory);
        }

        @Override
        public boolean supportsBitmapIndex(BitmapIndexSelector selector) {
            return false;
        }

        @Override
        public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) {
            return false;
        }

        @Override
        public VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory) {
            return theFilter.makeVectorMatcher(factory);
        }

        @Override
        public boolean canVectorizeMatcher(ColumnInspector inspector) {
            return theFilter.canVectorizeMatcher(inspector);
        }

        @Override
        public Set<String> getRequiredColumns() {
            return null;
        }

        @Override
        public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) {
            return false;
        }

        @Override
        public double estimateSelectivity(BitmapIndexSelector indexSelector) {
            return 1.0;
        }
    };
    try (final VectorCursor cursor = makeVectorCursor(postFilteringFilter)) {
        final SingleValueDimensionVectorSelector selector = cursor.getColumnSelectorFactory().makeSingleValueDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
        final List<String> values = new ArrayList<>();
        while (!cursor.isDone()) {
            final int[] rowVector = selector.getRowVector();
            for (int i = 0; i < cursor.getCurrentVectorSize(); i++) {
                values.add(selector.lookupName(rowVector[i]));
            }
            cursor.advance();
        }
        return values;
    }
}

Example 5 with BitmapIndexSelector

use of org.apache.druid.query.filter.BitmapIndexSelector in project druid by druid-io.

the class BaseFilterTest method selectColumnValuesMatchingFilterUsingPostFiltering.

private List<String> selectColumnValuesMatchingFilterUsingPostFiltering(final DimFilter filter, final String selectColumn) {
    final Filter theFilter = makeFilter(filter);
    final Filter postFilteringFilter = new Filter() {

        @Override
        public <T> T getBitmapResult(BitmapIndexSelector selector, BitmapResultFactory<T> bitmapResultFactory) {
            throw new UnsupportedOperationException();
        }

        @Override
        public ValueMatcher makeMatcher(ColumnSelectorFactory factory) {
            return theFilter.makeMatcher(factory);
        }

        @Override
        public boolean supportsBitmapIndex(BitmapIndexSelector selector) {
            return false;
        }

        @Override
        public boolean shouldUseBitmapIndex(BitmapIndexSelector selector) {
            return false;
        }

        @Override
        public boolean supportsSelectivityEstimation(ColumnSelector columnSelector, BitmapIndexSelector indexSelector) {
            return false;
        }

        @Override
        public Set<String> getRequiredColumns() {
            return Collections.emptySet();
        }

        @Override
        public double estimateSelectivity(BitmapIndexSelector indexSelector) {
            return 1.0;
        }
    };
    final Sequence<Cursor> cursors = makeCursorSequence(postFilteringFilter);
    Sequence<List<String>> seq = Sequences.map(cursors, cursor -> {
        final DimensionSelector selector = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec(selectColumn, selectColumn));
        final List<String> values = new ArrayList<>();
        while (!cursor.isDone()) {
            IndexedInts row = selector.getRow();
            Preconditions.checkState(row.size() == 1);
            values.add(selector.lookupName(row.get(0)));
            cursor.advance();
        }
        return values;
    });
    return seq.toList().get(0);
}