Examples with BitmapIndex org.apache.druid.segment.column.BitmapIndex used on opensource projects

Example 11 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class DimensionPredicateFilterBenchmark method setup.

@Setup
public void setup() {
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add(i - START_INT);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}

Example 12 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class ListFilteredVirtualColumn method getBitmapIndex.

@Override
@Nullable
public BitmapIndex getBitmapIndex(String columnName, ColumnSelector selector) {
    final ColumnHolder holder = selector.getColumnHolder(delegate.getDimension());
    if (holder == null) {
        return null;
    }
    final BitmapIndex underlyingIndex = holder.getBitmapIndex();
    if (underlyingIndex == null) {
        return null;
    }
    final IdMapping idMapping;
    if (allowList) {
        idMapping = ListFilteredDimensionSpec.buildAllowListIdMapping(values, underlyingIndex.getCardinality(), null, underlyingIndex::getValue);
    } else {
        idMapping = ListFilteredDimensionSpec.buildDenyListIdMapping(values, underlyingIndex.getCardinality(), underlyingIndex::getValue);
    }
    return new ListFilteredBitmapIndex(underlyingIndex, idMapping);
}

Example 13 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class FiltersTest method testEstimateSelectivityOfBitmapList.

@Test
public void testEstimateSelectivityOfBitmapList() {
    final int bitmapNum = 100;
    final List<ImmutableBitmap> bitmaps = Lists.newArrayListWithCapacity(bitmapNum);
    final BitmapIndex bitmapIndex = makeNonOverlappedBitmapIndexes(bitmapNum, bitmaps);
    final double estimated = Filters.estimateSelectivity(bitmapIndex, IntIteratorUtils.toIntList(IntIterators.fromTo(0, bitmapNum)), 10000);
    final double expected = 0.1;
    Assert.assertEquals(expected, estimated, 0.00001);
}

Example 14 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class Filters method shouldUseBitmapIndex.

/**
 * This method provides a "standard" implementation of {@link Filter#shouldUseBitmapIndex(BitmapIndexSelector)} which takes
 * a {@link Filter}, a {@link BitmapIndexSelector}, and {@link FilterTuning} to determine if:
 * a) the filter supports bitmap indexes for all required columns
 * b) the filter tuning specifies that it should use the index
 * c) the cardinality of the column is above the minimum threshold and below the maximum threshold to use the index
 *
 * If all these things are true, {@link org.apache.druid.segment.QueryableIndexStorageAdapter} will utilize the
 * indexes.
 */
public static boolean shouldUseBitmapIndex(Filter filter, BitmapIndexSelector indexSelector, @Nullable FilterTuning filterTuning) {
    final FilterTuning tuning = filterTuning != null ? filterTuning : FilterTuning.createDefault(filter, indexSelector);
    if (filter.supportsBitmapIndex(indexSelector) && tuning.getUseBitmapIndex()) {
        return filter.getRequiredColumns().stream().allMatch(column -> {
            final BitmapIndex index = indexSelector.getBitmapIndex(column);
            Preconditions.checkNotNull(index, "Column does not have a bitmap index");
            final int cardinality = index.getCardinality();
            return cardinality >= tuning.getMinCardinalityToUseBitmapIndex() && cardinality <= tuning.getMaxCardinalityToUseBitmapIndex();
        });
    }
    return false;
}

Example 15 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class Filters method estimateSelectivity.

/**
 * Return an estimated selectivity for bitmaps of all values matching the given predicate.
 *
 * @param dimension     dimension to look at
 * @param indexSelector bitmap selector
 * @param predicate     predicate to use
 *
 * @return estimated selectivity
 *
 * @see #matchPredicate(String, BitmapIndexSelector, BitmapResultFactory, Predicate)
 */
public static double estimateSelectivity(final String dimension, final BitmapIndexSelector indexSelector, final Predicate<String> predicate) {
    Preconditions.checkNotNull(dimension, "dimension");
    Preconditions.checkNotNull(indexSelector, "selector");
    Preconditions.checkNotNull(predicate, "predicate");
    // Missing dimension -> match all rows if the predicate matches null; match no rows otherwise
    try (final CloseableIndexed<String> dimValues = indexSelector.getDimensionValues(dimension)) {
        if (dimValues == null || dimValues.size() == 0) {
            return predicate.apply(null) ? 1. : 0.;
        }
        // Apply predicate to all dimension values and union the matching bitmaps
        final BitmapIndex bitmapIndex = indexSelector.getBitmapIndex(dimension);
        return estimateSelectivity(bitmapIndex, IntIteratorUtils.toIntList(makePredicateQualifyingIndexIterable(bitmapIndex, predicate, dimValues).iterator()), indexSelector.getNumRows());
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}