Search in sources :

Example 21 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class DimensionPredicateFilterBenchmark method setup.

@Setup
public void setup() {
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add(i - START_INT);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}
Also used : StringBitmapIndexColumnPartSupplier(org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) Function(com.google.common.base.Function) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory) Setup(org.openjdk.jmh.annotations.Setup)

Example 22 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class RangeBitmapBenchmark method setup.

@Setup(Level.Trial)
public void setup() throws IOException {
    switch(type) {
        case "concise":
            bitmapFactory = new ConciseBitmapFactory();
            break;
        case "roaring":
            bitmapFactory = new RoaringBitmapFactory();
            break;
        default:
            throw new IAE("Unknown bitmap type[%s]", type);
    }
    bitmaps = new ArrayList<>(numBitmaps);
    for (int i = 0; i < numBitmaps; ++i) {
        final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
        int k = 0;
        boolean fill = true;
        while (k < bitmapLength) {
            int runLength = (int) (bitmapLength * density) + RANDOM.nextInt((int) (bitmapLength * density));
            for (int j = k; fill && j < bitmapLength && j < k + runLength; ++j) {
                mutableBitmap.add(j);
            }
            k += runLength;
            fill = !fill;
        }
        for (k = bitmapLength / 2; k < bitmapLength / 2 + minIntersect; ++k) {
            mutableBitmap.add(k);
        }
        bitmaps.add(BitmapBenchmarkUtils.toOffheap(bitmapFactory.makeImmutableBitmap(mutableBitmap)));
    }
    final long totalSizeBytes = bitmaps.stream().mapToLong(bitmap -> bitmap.toBytes().length).sum();
    BitmapBenchmarkUtils.printSizeStats(type, density, bitmaps.size(), totalSizeBytes);
}
Also used : BenchmarkMode(org.openjdk.jmh.annotations.BenchmarkMode) Measurement(org.openjdk.jmh.annotations.Measurement) Blackhole(org.openjdk.jmh.infra.Blackhole) Scope(org.openjdk.jmh.annotations.Scope) Random(java.util.Random) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) Warmup(org.openjdk.jmh.annotations.Warmup) ArrayList(java.util.ArrayList) OutputTimeUnit(org.openjdk.jmh.annotations.OutputTimeUnit) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) IAE(org.apache.druid.java.util.common.IAE) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) Setup(org.openjdk.jmh.annotations.Setup) Mode(org.openjdk.jmh.annotations.Mode) Param(org.openjdk.jmh.annotations.Param) IOException(java.io.IOException) State(org.openjdk.jmh.annotations.State) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) Benchmark(org.openjdk.jmh.annotations.Benchmark) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) NullHandling(org.apache.druid.common.config.NullHandling) Level(org.openjdk.jmh.annotations.Level) Fork(org.openjdk.jmh.annotations.Fork) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) IAE(org.apache.druid.java.util.common.IAE) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) Setup(org.openjdk.jmh.annotations.Setup)

Example 23 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class UseIndexesStrategy method makeTimeFilteredBitmap.

static ImmutableBitmap makeTimeFilteredBitmap(final QueryableIndex index, final Segment segment, final Filter filter, final Interval interval) {
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final ImmutableBitmap baseFilter;
    if (filter == null) {
        baseFilter = null;
    } else {
        final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), VirtualColumns.EMPTY, index);
        Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter);
        baseFilter = filter.getBitmapIndex(selector);
    }
    final ImmutableBitmap timeFilteredBitmap;
    if (!interval.contains(segment.getDataInterval())) {
        final MutableBitmap timeBitmap = bitmapFactory.makeEmptyMutableBitmap();
        final ColumnHolder timeColumnHolder = index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME);
        try (final NumericColumn timeValues = (NumericColumn) timeColumnHolder.getColumn()) {
            int startIndex = Math.max(0, getStartIndexOfTime(timeValues, interval.getStartMillis(), true));
            int endIndex = Math.min(timeValues.length() - 1, getStartIndexOfTime(timeValues, interval.getEndMillis(), false));
            for (int i = startIndex; i <= endIndex; i++) {
                timeBitmap.add(i);
            }
            final ImmutableBitmap finalTimeBitmap = bitmapFactory.makeImmutableBitmap(timeBitmap);
            timeFilteredBitmap = (baseFilter == null) ? finalTimeBitmap : finalTimeBitmap.intersection(baseFilter);
        }
    } else {
        timeFilteredBitmap = baseFilter;
    }
    return timeFilteredBitmap;
}
Also used : ColumnSelectorBitmapIndexSelector(org.apache.druid.segment.ColumnSelectorBitmapIndexSelector) ColumnHolder(org.apache.druid.segment.column.ColumnHolder) NumericColumn(org.apache.druid.segment.column.NumericColumn) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) ColumnSelectorBitmapIndexSelector(org.apache.druid.segment.ColumnSelectorBitmapIndexSelector) BitmapIndexSelector(org.apache.druid.query.filter.BitmapIndexSelector) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory)

Example 24 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ImmutableRTreeTest method testEmptyConciseSet.

@Test
public void testEmptyConciseSet() {
    BitmapFactory bf = new ConciseBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    tree.insert(new float[] { 0.0f, 0.0f }, bf.makeEmptyMutableBitmap());
    ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
    Iterable<ImmutableBitmap> points = searchTree.search(new RadiusBound(new float[] { 0.0f, 0.0f }, 5));
    ImmutableBitmap finalSet = bf.union(points);
    Assert.assertEquals(finalSet.size(), 0);
}
Also used : ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RadiusBound(org.apache.druid.collections.spatial.search.RadiusBound) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) LinearGutmanSplitStrategy(org.apache.druid.collections.spatial.split.LinearGutmanSplitStrategy) Test(org.junit.Test)

Example 25 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ImmutableRTreeTest method testEmptyRoaringBitmap.

@Test
public void testEmptyRoaringBitmap() {
    BitmapFactory bf = new RoaringBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    tree.insert(new float[] { 0.0f, 0.0f }, bf.makeEmptyMutableBitmap());
    ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
    Iterable<ImmutableBitmap> points = searchTree.search(new RadiusBound(new float[] { 0.0f, 0.0f }, 5));
    ImmutableBitmap finalSet = bf.union(points);
    Assert.assertEquals(finalSet.size(), 0);
    Assert.assertTrue(finalSet.isEmpty());
}
Also used : RadiusBound(org.apache.druid.collections.spatial.search.RadiusBound) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) LinearGutmanSplitStrategy(org.apache.druid.collections.spatial.split.LinearGutmanSplitStrategy) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) Test(org.junit.Test)

Aggregations

BitmapFactory (org.apache.druid.collections.bitmap.BitmapFactory)36 RoaringBitmapFactory (org.apache.druid.collections.bitmap.RoaringBitmapFactory)32 ConciseBitmapFactory (org.apache.druid.collections.bitmap.ConciseBitmapFactory)28 ImmutableBitmap (org.apache.druid.collections.bitmap.ImmutableBitmap)26 Test (org.junit.Test)23 LinearGutmanSplitStrategy (org.apache.druid.collections.spatial.split.LinearGutmanSplitStrategy)20 Random (java.util.Random)16 IntIterator (org.roaringbitmap.IntIterator)15 ThreadLocalRandom (java.util.concurrent.ThreadLocalRandom)14 RadiusBound (org.apache.druid.collections.spatial.search.RadiusBound)13 MutableBitmap (org.apache.druid.collections.bitmap.MutableBitmap)8 BitmapIndex (org.apache.druid.segment.column.BitmapIndex)8 ColumnHolder (org.apache.druid.segment.column.ColumnHolder)5 BitmapSerdeFactory (org.apache.druid.segment.data.BitmapSerdeFactory)5 Setup (org.openjdk.jmh.annotations.Setup)5 RoaringBitmapSerdeFactory (org.apache.druid.segment.data.RoaringBitmapSerdeFactory)4 Function (com.google.common.base.Function)3 IOException (java.io.IOException)3 ArrayList (java.util.ArrayList)3 List (java.util.List)3