Examples with BitmapIndex io.druid.segment.column.BitmapIndex used on opensource projects

Example 6 with BitmapIndex

use of io.druid.segment.column.BitmapIndex in project druid by druid-io.

the class DimensionPredicateFilterBenchmark method setup.

@Setup
public void setup() throws IOException {
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new BitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add(i - START_INT);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new BitmapIndexSelector() {

        @Override
        public Indexed<String> getDimensionValues(String dimension) {
            return dictionary;
        }

        @Override
        public int getNumRows() {
            throw new UnsupportedOperationException();
        }

        @Override
        public BitmapFactory getBitmapFactory() {
            return bitmapFactory;
        }

        @Override
        public ImmutableBitmap getBitmapIndex(String dimension, String value) {
            return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
        }

        @Override
        public BitmapIndex getBitmapIndex(String dimension) {
            return bitmapIndex;
        }

        @Override
        public ImmutableRTree getSpatialIndex(String dimension) {
            throw new UnsupportedOperationException();
        }
    };
}

Example 7 with BitmapIndex

use of io.druid.segment.column.BitmapIndex in project druid by druid-io.

the class FiltersTest method testEstimateSelectivityOfBitmapList.

@Test
public void testEstimateSelectivityOfBitmapList() {
    final int bitmapNum = 100;
    final List<ImmutableBitmap> bitmaps = Lists.newArrayListWithCapacity(bitmapNum);
    final BitmapIndex bitmapIndex = makeNonOverlappedBitmapIndexes(bitmapNum, bitmaps);
    final double estimated = Filters.estimateSelectivity(bitmapIndex, IntIteratorUtils.toIntList(IntIterators.fromTo(0, bitmapNum)), 10000);
    final double expected = 0.1;
    assertEquals(expected, estimated, 0.00001);
}

Example 8 with BitmapIndex

use of io.druid.segment.column.BitmapIndex in project druid by druid-io.

the class ColumnSelectorBitmapIndexSelector method getBitmapIndex.

@Override
public ImmutableBitmap getBitmapIndex(String dimension, String value) {
    if (isFilterableVirtualColumn(dimension)) {
        // Virtual columns don't have dictionaries or indexes.
        return null;
    }
    final Column column = index.getColumn(dimension);
    if (column == null || !columnSupportsFiltering(column)) {
        if (Strings.isNullOrEmpty(value)) {
            return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows());
        } else {
            return bitmapFactory.makeEmptyImmutableBitmap();
        }
    }
    if (!column.getCapabilities().hasBitmapIndexes()) {
        return null;
    }
    final BitmapIndex bitmapIndex = column.getBitmapIndex();
    return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
}

Example 9 with BitmapIndex

use of io.druid.segment.column.BitmapIndex in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try {
                final JsonGenerator jg = objectMapper.getFactory().createGenerator(out);
                jg.writeStartObject();
                jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                jg.writeFieldName("bitmaps");
                jg.writeStartObject();
                for (final String columnName : columnNames) {
                    final Column column = index.getColumn(columnName);
                    final BitmapIndex bitmapIndex = column.getBitmapIndex();
                    if (bitmapIndex == null) {
                        jg.writeNullField(columnName);
                    } else {
                        jg.writeFieldName(columnName);
                        jg.writeStartObject();
                        for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                            jg.writeFieldName(Strings.nullToEmpty(bitmapIndex.getValue(i)));
                            final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                            if (decompressBitmaps) {
                                jg.writeStartArray();
                                final IntIterator iterator = bitmap.iterator();
                                while (iterator.hasNext()) {
                                    final int rowNum = iterator.next();
                                    jg.writeNumber(rowNum);
                                }
                                jg.writeEndArray();
                            } else {
                                jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap));
                            }
                        }
                        jg.writeEndObject();
                    }
                }
                jg.writeEndObject();
                jg.writeEndObject();
                jg.close();
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
            return null;
        }
    });
}

Example 10 with BitmapIndex

use of io.druid.segment.column.BitmapIndex in project druid by druid-io.

the class Filters method matchPredicateNoUnion.

/**
   * Return an iterable of bitmaps for all values matching a particular predicate. Unioning these bitmaps
   * yields the same result that {@link #matchPredicate(String, BitmapIndexSelector, Predicate)} would have
   * returned.
   *
   * @param dimension dimension to look at
   * @param selector  bitmap selector
   * @param predicate predicate to use
   *
   * @return iterable of bitmaps of matching rows
   */
public static Iterable<ImmutableBitmap> matchPredicateNoUnion(final String dimension, final BitmapIndexSelector selector, final Predicate<String> predicate) {
    Preconditions.checkNotNull(dimension, "dimension");
    Preconditions.checkNotNull(selector, "selector");
    Preconditions.checkNotNull(predicate, "predicate");
    // Missing dimension -> match all rows if the predicate matches null; match no rows otherwise
    final Indexed<String> dimValues = selector.getDimensionValues(dimension);
    if (dimValues == null || dimValues.size() == 0) {
        return ImmutableList.of(predicate.apply(null) ? allTrue(selector) : allFalse(selector));
    }
    // Apply predicate to all dimension values and union the matching bitmaps
    final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension);
    return makePredicateQualifyingBitmapIterable(bitmapIndex, predicate, dimValues);
}