Search in sources :

Example 26 with ImmutableBitmap

use of io.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class LikeFilterBenchmark method matchLikeEquals.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void matchLikeEquals(Blackhole blackhole) {
    final ImmutableBitmap bitmapIndex = LIKE_EQUALS.getBitmapIndex(selector);
    blackhole.consume(bitmapIndex);
}
Also used : ImmutableBitmap(io.druid.collections.bitmap.ImmutableBitmap) BenchmarkMode(org.openjdk.jmh.annotations.BenchmarkMode) Benchmark(org.openjdk.jmh.annotations.Benchmark) OutputTimeUnit(org.openjdk.jmh.annotations.OutputTimeUnit)

Example 27 with ImmutableBitmap

use of io.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try {
                final JsonGenerator jg = objectMapper.getFactory().createGenerator(out);
                jg.writeStartObject();
                jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                jg.writeFieldName("bitmaps");
                jg.writeStartObject();
                for (final String columnName : columnNames) {
                    final Column column = index.getColumn(columnName);
                    final BitmapIndex bitmapIndex = column.getBitmapIndex();
                    if (bitmapIndex == null) {
                        jg.writeNullField(columnName);
                    } else {
                        jg.writeFieldName(columnName);
                        jg.writeStartObject();
                        for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                            jg.writeFieldName(Strings.nullToEmpty(bitmapIndex.getValue(i)));
                            final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                            if (decompressBitmaps) {
                                jg.writeStartArray();
                                final IntIterator iterator = bitmap.iterator();
                                while (iterator.hasNext()) {
                                    final int rowNum = iterator.next();
                                    jg.writeNumber(rowNum);
                                }
                                jg.writeEndArray();
                            } else {
                                jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap));
                            }
                        }
                        jg.writeEndObject();
                    }
                }
                jg.writeEndObject();
                jg.writeEndObject();
                jg.close();
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
            return null;
        }
    });
}
Also used : ConciseBitmapFactory(io.druid.collections.bitmap.ConciseBitmapFactory) IntIterator(org.roaringbitmap.IntIterator) ImmutableBitmap(io.druid.collections.bitmap.ImmutableBitmap) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) BitmapIndex(io.druid.segment.column.BitmapIndex) Json(io.druid.guice.annotations.Json) IOException(java.io.IOException) RoaringBitmapSerdeFactory(io.druid.segment.data.RoaringBitmapSerdeFactory) ConciseBitmapSerdeFactory(io.druid.segment.data.ConciseBitmapSerdeFactory) Column(io.druid.segment.column.Column) JsonGenerator(com.fasterxml.jackson.core.JsonGenerator) ISE(io.druid.java.util.common.ISE) ConciseBitmapFactory(io.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(io.druid.collections.bitmap.RoaringBitmapFactory) BitmapFactory(io.druid.collections.bitmap.BitmapFactory) RoaringBitmapFactory(io.druid.collections.bitmap.RoaringBitmapFactory) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) RoaringBitmapSerdeFactory(io.druid.segment.data.RoaringBitmapSerdeFactory) ConciseBitmapSerdeFactory(io.druid.segment.data.ConciseBitmapSerdeFactory) BitmapSerdeFactory(io.druid.segment.data.BitmapSerdeFactory)

Example 28 with ImmutableBitmap

use of io.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class BitmapOffsetTest method testSanity.

@Test
public void testSanity() throws Exception {
    MutableBitmap mutable = factory.makeEmptyMutableBitmap();
    for (int val : TEST_VALS) {
        mutable.add(val);
    }
    ImmutableBitmap bitmap = factory.makeImmutableBitmap(mutable);
    final BitmapOffset offset = BitmapOffset.of(bitmap, descending, bitmap.size());
    final int[] expected = descending ? TEST_VALS_FLIP : TEST_VALS;
    int count = 0;
    while (offset.withinBounds()) {
        Assert.assertEquals(expected[count], offset.getOffset());
        int cloneCount = count;
        Offset clonedOffset = offset.clone();
        while (clonedOffset.withinBounds()) {
            Assert.assertEquals(expected[cloneCount], clonedOffset.getOffset());
            ++cloneCount;
            clonedOffset.increment();
        }
        ++count;
        offset.increment();
    }
    Assert.assertEquals(count, expected.length);
}
Also used : ImmutableBitmap(io.druid.collections.bitmap.ImmutableBitmap) MutableBitmap(io.druid.collections.bitmap.MutableBitmap) Offset(io.druid.segment.data.Offset) Test(org.junit.Test)

Example 29 with ImmutableBitmap

use of io.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class BitmapCreationBenchmark method testFromImmutableByteArray.

@BenchmarkOptions(warmupRounds = 10, benchmarkRounds = 1000)
@Test
public void testFromImmutableByteArray() {
    ImmutableBitmap immutableBitmap = factory.mapImmutableBitmap(baseByteBuffer);
    Assert.assertEquals(numBits, immutableBitmap.size());
}
Also used : ImmutableBitmap(io.druid.collections.bitmap.ImmutableBitmap) Test(org.junit.Test) BenchmarkOptions(com.carrotsearch.junitbenchmarks.BenchmarkOptions)

Example 30 with ImmutableBitmap

use of io.druid.collections.bitmap.ImmutableBitmap in project druid by druid-io.

the class Filters method estimateSelectivity.

/**
   * Return an estimated selectivity for bitmaps for the dimension values given by dimValueIndexes.
   *
   * @param bitmapIndex  bitmap index
   * @param bitmaps      bitmaps to extract, by index
   * @param totalNumRows number of rows in the column associated with this bitmap index
   *
   * @return estimated selectivity
   */
public static double estimateSelectivity(final BitmapIndex bitmapIndex, final IntList bitmaps, final long totalNumRows) {
    long numMatchedRows = 0;
    for (int i = 0; i < bitmaps.size(); i++) {
        final ImmutableBitmap bitmap = bitmapIndex.getBitmap(bitmaps.get(i));
        numMatchedRows += bitmap.size();
    }
    return Math.min(1., (double) numMatchedRows / totalNumRows);
}
Also used : ImmutableBitmap(io.druid.collections.bitmap.ImmutableBitmap)

Aggregations

ImmutableBitmap (io.druid.collections.bitmap.ImmutableBitmap)51 BitmapFactory (io.druid.collections.bitmap.BitmapFactory)23 Test (org.junit.Test)23 RoaringBitmapFactory (io.druid.collections.bitmap.RoaringBitmapFactory)22 ConciseBitmapFactory (io.druid.collections.bitmap.ConciseBitmapFactory)19 LinearGutmanSplitStrategy (io.druid.collections.spatial.split.LinearGutmanSplitStrategy)18 IntIterator (org.roaringbitmap.IntIterator)16 RadiusBound (io.druid.collections.spatial.search.RadiusBound)13 Benchmark (org.openjdk.jmh.annotations.Benchmark)13 Random (java.util.Random)12 BenchmarkMode (org.openjdk.jmh.annotations.BenchmarkMode)12 OutputTimeUnit (org.openjdk.jmh.annotations.OutputTimeUnit)12 MutableBitmap (io.druid.collections.bitmap.MutableBitmap)7 Filter (io.druid.query.filter.Filter)6 BitmapIndexSelector (io.druid.query.filter.BitmapIndexSelector)5 BitmapIndex (io.druid.segment.column.BitmapIndex)5 ImmutableRTree (io.druid.collections.spatial.ImmutableRTree)4 BitmapSerdeFactory (io.druid.segment.data.BitmapSerdeFactory)4 RoaringBitmapSerdeFactory (io.druid.segment.data.RoaringBitmapSerdeFactory)4 Function (com.google.common.base.Function)3