use of org.apache.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class DumpSegment method runBitmaps.
private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
final BitmapSerdeFactory bitmapSerdeFactory;
if (bitmapFactory instanceof ConciseBitmapFactory) {
bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
} else if (bitmapFactory instanceof RoaringBitmapFactory) {
bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
} else {
throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
}
final List<String> columnNames = getColumnsToInclude(index);
withOutputStream(new Function<OutputStream, Object>() {
@Override
public Object apply(final OutputStream out) {
try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
jg.writeStartObject();
{
jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
jg.writeFieldName("bitmaps");
jg.writeStartObject();
{
for (final String columnName : columnNames) {
final ColumnHolder columnHolder = index.getColumnHolder(columnName);
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
if (bitmapIndex == null) {
jg.writeNullField(columnName);
} else {
jg.writeFieldName(columnName);
jg.writeStartObject();
for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
String val = bitmapIndex.getValue(i);
// respect nulls if they are present in the dictionary
jg.writeFieldName(val == null ? "null" : val);
final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
if (decompressBitmaps) {
jg.writeStartArray();
final IntIterator iterator = bitmap.iterator();
while (iterator.hasNext()) {
final int rowNum = iterator.next();
jg.writeNumber(rowNum);
}
jg.writeEndArray();
} else {
byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap);
if (bytes != null) {
jg.writeBinary(bytes);
}
}
}
jg.writeEndObject();
}
}
}
jg.writeEndObject();
}
jg.writeEndObject();
} catch (IOException e) {
throw new RuntimeException(e);
}
return null;
}
});
}
use of org.apache.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class ListFilteredVirtualColumnSelectorTest method testFilterListFilteredVirtualColumnDenyIndex.
@Test
public void testFilterListFilteredVirtualColumnDenyIndex() {
ListFilteredVirtualColumn virtualColumn = new ListFilteredVirtualColumn(DENY_VIRTUAL_NAME, new DefaultDimensionSpec(COLUMN_NAME, COLUMN_NAME, ColumnType.STRING), ImmutableSet.of("a", "b"), false);
ColumnSelector selector = EasyMock.createMock(ColumnSelector.class);
ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
BitmapIndex index = EasyMock.createMock(BitmapIndex.class);
ImmutableBitmap bitmap = EasyMock.createMock(ImmutableBitmap.class);
BitmapFactory bitmapFactory = EasyMock.createMock(BitmapFactory.class);
EasyMock.expect(selector.getColumnHolder(COLUMN_NAME)).andReturn(holder).atLeastOnce();
EasyMock.expect(holder.getBitmapIndex()).andReturn(index).atLeastOnce();
EasyMock.expect(index.getCardinality()).andReturn(3).atLeastOnce();
EasyMock.expect(index.getValue(0)).andReturn("a").atLeastOnce();
EasyMock.expect(index.getValue(1)).andReturn("b").atLeastOnce();
EasyMock.expect(index.getValue(2)).andReturn("c").atLeastOnce();
EasyMock.expect(index.getBitmap(0)).andReturn(bitmap).once();
EasyMock.expect(index.getBitmapFactory()).andReturn(bitmapFactory).once();
EasyMock.expect(index.hasNulls()).andReturn(true).once();
EasyMock.replay(selector, holder, index, bitmap, bitmapFactory);
ColumnSelectorBitmapIndexSelector bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(new RoaringBitmapFactory(), VirtualColumns.create(Collections.singletonList(virtualColumn)), selector);
SelectorFilter filter = new SelectorFilter(DENY_VIRTUAL_NAME, "c");
Assert.assertTrue(filter.shouldUseBitmapIndex(bitmapIndexSelector));
BitmapIndex listFilteredIndex = bitmapIndexSelector.getBitmapIndex(DENY_VIRTUAL_NAME);
Assert.assertEquals(-1, listFilteredIndex.getIndex("a"));
Assert.assertEquals(-1, listFilteredIndex.getIndex("b"));
Assert.assertEquals(0, listFilteredIndex.getIndex("c"));
Assert.assertEquals(1, listFilteredIndex.getCardinality());
Assert.assertEquals(bitmap, listFilteredIndex.getBitmap(1));
Assert.assertEquals(bitmapFactory, listFilteredIndex.getBitmapFactory());
Assert.assertTrue(listFilteredIndex.hasNulls());
EasyMock.verify(selector, holder, index, bitmap, bitmapFactory);
}
use of org.apache.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class IndexMergerTestBase method testMismatchedDimensions.
@Test
public void testMismatchedDimensions() throws IOException {
IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
index1.add(new MapBasedInputRow(1L, Arrays.asList("d1", "d2"), ImmutableMap.of("d1", "a", "d2", "z", "A", 1)));
closer.closeLater(index1);
IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
index2.add(new MapBasedInputRow(1L, Arrays.asList("d1", "d2"), ImmutableMap.of("d1", "a", "d2", "z", "A", 2, "C", 100)));
closer.closeLater(index2);
Interval interval = new Interval(DateTimes.EPOCH, DateTimes.nowUtc());
RoaringBitmapFactory factory = new RoaringBitmapFactory();
List<IndexableAdapter> toMerge = Arrays.asList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory));
final File tmpDirMerged = temporaryFolder.newFolder();
indexMerger.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") }, tmpDirMerged, indexSpec, -1);
}
use of org.apache.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class IndexMergerTestBase method testMismatchedMetrics.
@Test
public void testMismatchedMetrics() throws IOException {
IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
closer.closeLater(index1);
IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
closer.closeLater(index2);
IncrementalIndex index3 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("B", "B") });
closer.closeLater(index3);
IncrementalIndex index4 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("B", "B") });
closer.closeLater(index4);
IncrementalIndex index5 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("B", "B") });
closer.closeLater(index5);
Interval interval = new Interval(DateTimes.EPOCH, DateTimes.nowUtc());
RoaringBitmapFactory factory = new RoaringBitmapFactory();
List<IndexableAdapter> toMerge = Arrays.asList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory), new IncrementalIndexAdapter(interval, index3, factory), new IncrementalIndexAdapter(interval, index4, factory), new IncrementalIndexAdapter(interval, index5, factory));
final File tmpDirMerged = temporaryFolder.newFolder();
File merged = indexMerger.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("B", "B"), new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("D", "D") }, tmpDirMerged, indexSpec, -1);
// Since D was not present in any of the indices, it is not present in the output
final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(indexIO.loadIndex(merged)));
Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
}
use of org.apache.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class ImmutableRTreeTest method testToAndFromByteBufferRoaring.
@Test
public void testToAndFromByteBufferRoaring() {
BitmapFactory bf = new RoaringBitmapFactory();
RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
tree.insert(new float[] { 0, 0 }, 1);
tree.insert(new float[] { 1, 1 }, 2);
tree.insert(new float[] { 2, 2 }, 3);
tree.insert(new float[] { 3, 3 }, 4);
tree.insert(new float[] { 4, 4 }, 5);
ImmutableRTree firstTree = ImmutableRTree.newImmutableFromMutable(tree);
ByteBuffer buffer = ByteBuffer.wrap(firstTree.toBytes());
ImmutableRTree secondTree = new ImmutableRTree(buffer, bf);
Iterable<ImmutableBitmap> points = secondTree.search(new RadiusBound(new float[] { 0, 0 }, 10));
ImmutableBitmap finalSet = bf.union(points);
Assert.assertTrue(finalSet.size() >= 5);
Set<Integer> expected = Sets.newHashSet(1, 2, 3, 4, 5);
IntIterator iter = finalSet.iterator();
while (iter.hasNext()) {
Assert.assertTrue(expected.contains(iter.next()));
}
}
Aggregations