Search in sources :

Example 1 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class FiltersTest method makeNonOverlappedBitmapIndexes.

private static BitmapIndex makeNonOverlappedBitmapIndexes(final int bitmapNum, final List<ImmutableBitmap> bitmaps) {
    final BitmapIndex bitmapIndex = getBitmapIndex(bitmaps);
    final BitmapFactory factory = bitmapIndex.getBitmapFactory();
    for (int i = 0; i < bitmapNum; i++) {
        final MutableBitmap mutableBitmap = factory.makeEmptyMutableBitmap();
        for (int j = 0; j < 10; j++) {
            mutableBitmap.add(i * 10 + j);
        }
        bitmaps.add(factory.makeImmutableBitmap(mutableBitmap));
    }
    return bitmapIndex;
}
Also used : MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory)

Example 2 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
                jg.writeStartObject();
                {
                    jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                    jg.writeFieldName("bitmaps");
                    jg.writeStartObject();
                    {
                        for (final String columnName : columnNames) {
                            final ColumnHolder columnHolder = index.getColumnHolder(columnName);
                            final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
                            if (bitmapIndex == null) {
                                jg.writeNullField(columnName);
                            } else {
                                jg.writeFieldName(columnName);
                                jg.writeStartObject();
                                for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                                    String val = bitmapIndex.getValue(i);
                                    // respect nulls if they are present in the dictionary
                                    jg.writeFieldName(val == null ? "null" : val);
                                    final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                                    if (decompressBitmaps) {
                                        jg.writeStartArray();
                                        final IntIterator iterator = bitmap.iterator();
                                        while (iterator.hasNext()) {
                                            final int rowNum = iterator.next();
                                            jg.writeNumber(rowNum);
                                        }
                                        jg.writeEndArray();
                                    } else {
                                        byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap);
                                        if (bytes != null) {
                                            jg.writeBinary(bytes);
                                        }
                                    }
                                }
                                jg.writeEndObject();
                            }
                        }
                    }
                    jg.writeEndObject();
                }
                jg.writeEndObject();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            return null;
        }
    });
}
Also used : ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) ColumnHolder(org.apache.druid.segment.column.ColumnHolder) IntIterator(org.roaringbitmap.IntIterator) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) Json(org.apache.druid.guice.annotations.Json) IOException(java.io.IOException) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) JsonGenerator(com.fasterxml.jackson.core.JsonGenerator) ISE(org.apache.druid.java.util.common.ISE) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory)

Example 3 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ListFilteredVirtualColumnSelectorTest method testFilterListFilteredVirtualColumnDenyIndex.

@Test
public void testFilterListFilteredVirtualColumnDenyIndex() {
    ListFilteredVirtualColumn virtualColumn = new ListFilteredVirtualColumn(DENY_VIRTUAL_NAME, new DefaultDimensionSpec(COLUMN_NAME, COLUMN_NAME, ColumnType.STRING), ImmutableSet.of("a", "b"), false);
    ColumnSelector selector = EasyMock.createMock(ColumnSelector.class);
    ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
    BitmapIndex index = EasyMock.createMock(BitmapIndex.class);
    ImmutableBitmap bitmap = EasyMock.createMock(ImmutableBitmap.class);
    BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
    EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
    EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
    EasyMock.expect(index.getCardinality()).andReturn(3).atLeastOnce();
    EasyMock.expect(index.getValue(0)).andReturn("a").atLeastOnce();
    EasyMock.expect(index.getValue(1)).andReturn("b").atLeastOnce();
    EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
    EasyMock.expect(index.getBitmap(0)).andReturn(bitmap).once();
    EasyMock.expect(index.getBitmapFactory()).andReturn(bitmapFactory).once();
    EasyMock.expect(index.hasNulls()).andReturn(true).once();
    EasyMock.replay(selector, holder, index, bitmap, bitmapFactory);
    ColumnSelectorBitmapIndexSelector bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(new RoaringBitmapFactory(), VirtualColumns.create(Collections.singletonList(virtualColumn)), selector);
    SelectorFilter filter = new SelectorFilter(DENY_VIRTUAL_NAME, "c");
    Assert.assertTrue(filter.shouldUseBitmapIndex(bitmapIndexSelector));
    BitmapIndex listFilteredIndex = bitmapIndexSelector.getBitmapIndex(DENY_VIRTUAL_NAME);
    Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
    Assert.assertEquals(-1, listFilteredIndex.getIndex("b"));
    Assert.assertEquals(0, listFilteredIndex.getIndex("c"));
    Assert.assertEquals(1, listFilteredIndex.getCardinality());
    Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
    Assert.assertEquals(bitmapFactory, listFilteredIndex.getBitmapFactory());
    Assert.assertTrue(listFilteredIndex.hasNulls());
    EasyMock.verify(selector, holder, index, bitmap, bitmapFactory);
}
Also used : ColumnSelectorBitmapIndexSelector(org.apache.druid.segment.ColumnSelectorBitmapIndexSelector) ColumnHolder(org.apache.druid.segment.column.ColumnHolder) SelectorFilter(org.apache.druid.segment.filter.SelectorFilter) ColumnSelector(org.apache.druid.segment.ColumnSelector) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 4 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ImmutableRTreeTest method testToAndFromByteBufferRoaring.

@Test
public void testToAndFromByteBufferRoaring() {
    BitmapFactory bf = new RoaringBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    tree.insert(new float[] { 0, 0 }, 1);
    tree.insert(new float[] { 1, 1 }, 2);
    tree.insert(new float[] { 2, 2 }, 3);
    tree.insert(new float[] { 3, 3 }, 4);
    tree.insert(new float[] { 4, 4 }, 5);
    ImmutableRTree firstTree = ImmutableRTree.newImmutableFromMutable(tree);
    ByteBuffer buffer = ByteBuffer.wrap(firstTree.toBytes());
    ImmutableRTree secondTree = new ImmutableRTree(buffer, bf);
    Iterable<ImmutableBitmap> points = secondTree.search(new RadiusBound(new float[] { 0, 0 }, 10));
    ImmutableBitmap finalSet = bf.union(points);
    Assert.assertTrue(finalSet.size() >= 5);
    Set<Integer> expected = Sets.newHashSet(1, 2, 3, 4, 5);
    IntIterator iter = finalSet.iterator();
    while (iter.hasNext()) {
        Assert.assertTrue(expected.contains(iter.next()));
    }
}
Also used : IntIterator(org.roaringbitmap.IntIterator) RadiusBound(org.apache.druid.collections.spatial.search.RadiusBound) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) LinearGutmanSplitStrategy(org.apache.druid.collections.spatial.split.LinearGutmanSplitStrategy) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) ByteBuffer(java.nio.ByteBuffer) Test(org.junit.Test)

Example 5 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ImmutableRTreeTest method testSearchWithSplitLimitedBoundRoaring.

@Test
public void testSearchWithSplitLimitedBoundRoaring() {
    BitmapFactory bf = new RoaringBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    tree.insert(new float[] { 0, 0 }, 1);
    tree.insert(new float[] { 1, 3 }, 2);
    tree.insert(new float[] { 4, 2 }, 3);
    tree.insert(new float[] { 5, 0 }, 4);
    tree.insert(new float[] { -4, -3 }, 5);
    Random rand = ThreadLocalRandom.current();
    for (int i = 0; i < 4995; i++) {
        tree.insert(new float[] { (float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0) }, i);
    }
    ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
    Iterable<ImmutableBitmap> points = searchTree.search(new RadiusBound(new float[] { 0, 0 }, 5, 2));
    ImmutableBitmap finalSet = bf.union(points);
    Assert.assertTrue(finalSet.size() >= 5);
    Set<Integer> expected = Sets.newHashSet(1, 2, 3, 4, 5);
    IntIterator iter = finalSet.iterator();
    while (iter.hasNext()) {
        Assert.assertTrue(expected.contains(iter.next()));
    }
}
Also used : IntIterator(org.roaringbitmap.IntIterator) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) LinearGutmanSplitStrategy(org.apache.druid.collections.spatial.split.LinearGutmanSplitStrategy) Random(java.util.Random) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) RadiusBound(org.apache.druid.collections.spatial.search.RadiusBound) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) Test(org.junit.Test)

Aggregations

BitmapFactory (org.apache.druid.collections.bitmap.BitmapFactory)36 RoaringBitmapFactory (org.apache.druid.collections.bitmap.RoaringBitmapFactory)32 ConciseBitmapFactory (org.apache.druid.collections.bitmap.ConciseBitmapFactory)28 ImmutableBitmap (org.apache.druid.collections.bitmap.ImmutableBitmap)26 Test (org.junit.Test)23 LinearGutmanSplitStrategy (org.apache.druid.collections.spatial.split.LinearGutmanSplitStrategy)20 Random (java.util.Random)16 IntIterator (org.roaringbitmap.IntIterator)15 ThreadLocalRandom (java.util.concurrent.ThreadLocalRandom)14 RadiusBound (org.apache.druid.collections.spatial.search.RadiusBound)13 MutableBitmap (org.apache.druid.collections.bitmap.MutableBitmap)8 BitmapIndex (org.apache.druid.segment.column.BitmapIndex)8 ColumnHolder (org.apache.druid.segment.column.ColumnHolder)5 BitmapSerdeFactory (org.apache.druid.segment.data.BitmapSerdeFactory)5 Setup (org.openjdk.jmh.annotations.Setup)5 RoaringBitmapSerdeFactory (org.apache.druid.segment.data.RoaringBitmapSerdeFactory)4 Function (com.google.common.base.Function)3 IOException (java.io.IOException)3 ArrayList (java.util.ArrayList)3 List (java.util.List)3