Examples with BitmapIndex org.apache.druid.segment.column.BitmapIndex used on opensource projects

Example 6 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class ColumnSelectorBitmapIndexSelector method getBitmapIndex.

@Override
public ImmutableBitmap getBitmapIndex(String dimension, String value) {
    if (isVirtualColumn(dimension)) {
        BitmapIndex idx = virtualColumns.getBitmapIndex(dimension, index);
        if (idx == null) {
            return null;
        }
        return idx.getBitmap(idx.getIndex(value));
    }
    final ColumnHolder columnHolder = index.getColumnHolder(dimension);
    if (columnHolder == null || !columnHolder.getCapabilities().isFilterable()) {
        if (NullHandling.isNullOrEquivalent(value)) {
            return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows());
        } else {
            return bitmapFactory.makeEmptyImmutableBitmap();
        }
    }
    if (!columnHolder.getCapabilities().hasBitmapIndexes() || !columnHolder.getCapabilities().is(ValueType.STRING)) {
        // other types of dictionary encoded columns
        return null;
    }
    final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
    return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
}

Example 7 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class ListFilteredVirtualColumnSelectorTest method testFilterListFilteredVirtualColumnDenyIndex.

@Test
public void testFilterListFilteredVirtualColumnDenyIndex() {
    ListFilteredVirtualColumn virtualColumn = new ListFilteredVirtualColumn(DENY_VIRTUAL_NAME, new DefaultDimensionSpec(COLUMN_NAME, COLUMN_NAME, ColumnType.STRING), ImmutableSet.of("a", "b"), false);
    ColumnSelector selector = EasyMock.createMock(ColumnSelector.class);
    ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
    BitmapIndex index = EasyMock.createMock(BitmapIndex.class);
    ImmutableBitmap bitmap = EasyMock.createMock(ImmutableBitmap.class);
    BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
    EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
    EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
    EasyMock.expect(index.getCardinality()).andReturn(3).atLeastOnce();
    EasyMock.expect(index.getValue(0)).andReturn("a").atLeastOnce();
    EasyMock.expect(index.getValue(1)).andReturn("b").atLeastOnce();
    EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
    EasyMock.expect(index.getBitmap(0)).andReturn(bitmap).once();
    EasyMock.expect(index.getBitmapFactory()).andReturn(bitmapFactory).once();
    EasyMock.expect(index.hasNulls()).andReturn(true).once();
    EasyMock.replay(selector, holder, index, bitmap, bitmapFactory);
    ColumnSelectorBitmapIndexSelector bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(new RoaringBitmapFactory(), VirtualColumns.create(Collections.singletonList(virtualColumn)), selector);
    SelectorFilter filter = new SelectorFilter(DENY_VIRTUAL_NAME, "c");
    Assert.assertTrue(filter.shouldUseBitmapIndex(bitmapIndexSelector));
    BitmapIndex listFilteredIndex = bitmapIndexSelector.getBitmapIndex(DENY_VIRTUAL_NAME);
    Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
    Assert.assertEquals(-1, listFilteredIndex.getIndex("b"));
    Assert.assertEquals(0, listFilteredIndex.getIndex("c"));
    Assert.assertEquals(1, listFilteredIndex.getCardinality());
    Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
    Assert.assertEquals(bitmapFactory, listFilteredIndex.getBitmapFactory());
    Assert.assertTrue(listFilteredIndex.hasNulls());
    EasyMock.verify(selector, holder, index, bitmap, bitmapFactory);
}

Example 8 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class SegmentAnalyzer method analyzeStringColumn.

private ColumnAnalysis analyzeStringColumn(final ColumnCapabilities capabilities, final ColumnHolder columnHolder) {
    Comparable min = null;
    Comparable max = null;
    long size = 0;
    final int cardinality;
    if (capabilities.hasBitmapIndexes()) {
        final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
        cardinality = bitmapIndex.getCardinality();
        if (analyzingSize()) {
            for (int i = 0; i < cardinality; ++i) {
                String value = bitmapIndex.getValue(i);
                if (value != null) {
                    size += StringUtils.estimatedBinaryLengthAsUTF8(value) * ((long) bitmapIndex.getBitmap(bitmapIndex.getIndex(value)).size());
                }
            }
        }
        if (analyzingMinMax() && cardinality > 0) {
            min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0));
            max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1));
        }
    } else if (capabilities.isDictionaryEncoded().isTrue()) {
        // fallback if no bitmap index
        try (BaseColumn column = columnHolder.getColumn()) {
            DictionaryEncodedColumn<String> theColumn = (DictionaryEncodedColumn<String>) column;
            cardinality = theColumn.getCardinality();
            if (analyzingMinMax() && cardinality > 0) {
                min = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(0));
                max = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(cardinality - 1));
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    } else {
        cardinality = 0;
    }
    return new ColumnAnalysis(capabilities.toColumnType(), capabilities.getType().name(), capabilities.hasMultipleValues().isTrue(), // if we don't know for sure, then we should plan to check for nulls
    capabilities.hasNulls().isMaybeTrue(), size, analyzingCardinality() ? cardinality : 0, min, max, null);
}

Example 9 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class LikeFilterBenchmark method setup.

@Setup
public void setup() {
    step = (END_INT - START_INT) / cardinality;
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add((i - START_INT) / step);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}

Example 10 with BitmapIndex

use of org.apache.druid.segment.column.BitmapIndex in project druid by druid-io.

the class BoundFilterBenchmark method setup.

@Setup
public void setup() {
    step = (END_INT - START_INT) / cardinality;
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(i -> i.toString()), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add((i - START_INT) / step);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}