use of io.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class IndexMergerTest method testMismatchedMetrics.
@Test
public void testMismatchedMetrics() throws IOException {
IncrementalIndex index1 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A") });
closer.closeLater(index1);
IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
closer.closeLater(index2);
IncrementalIndex index3 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("B", "B") });
closer.closeLater(index3);
IncrementalIndex index4 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("B", "B") });
closer.closeLater(index4);
IncrementalIndex index5 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("B", "B") });
closer.closeLater(index5);
Interval interval = new Interval(0, new DateTime().getMillis());
RoaringBitmapFactory factory = new RoaringBitmapFactory();
ArrayList<IndexableAdapter> toMerge = Lists.<IndexableAdapter>newArrayList(new IncrementalIndexAdapter(interval, index1, factory), new IncrementalIndexAdapter(interval, index2, factory), new IncrementalIndexAdapter(interval, index3, factory), new IncrementalIndexAdapter(interval, index4, factory), new IncrementalIndexAdapter(interval, index5, factory));
final File tmpDirMerged = temporaryFolder.newFolder();
File merged = INDEX_MERGER.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("B", "B"), new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("D", "D") }, tmpDirMerged, indexSpec);
// Since D was not present in any of the indices, it is not present in the output
final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex(merged)));
Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
}
use of io.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class IndexMergerTest method testMismatchedMetricsVarying.
@Test(expected = IAE.class)
public void testMismatchedMetricsVarying() throws IOException {
IncrementalIndex index2 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("C", "C") });
closer.closeLater(index2);
IncrementalIndex index5 = IncrementalIndexTest.createIndex(new AggregatorFactory[] { new LongSumAggregatorFactory("C", "C"), new LongSumAggregatorFactory("B", "B") });
closer.closeLater(index5);
Interval interval = new Interval(0, new DateTime().getMillis());
RoaringBitmapFactory factory = new RoaringBitmapFactory();
ArrayList<IndexableAdapter> toMerge = Lists.<IndexableAdapter>newArrayList(new IncrementalIndexAdapter(interval, index2, factory));
final File tmpDirMerged = temporaryFolder.newFolder();
final File merged = INDEX_MERGER.merge(toMerge, true, new AggregatorFactory[] { new LongSumAggregatorFactory("B", "B"), new LongSumAggregatorFactory("A", "A"), new LongSumAggregatorFactory("D", "D") }, tmpDirMerged, indexSpec);
final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(closer.closeLater(INDEX_IO.loadIndex(merged)));
Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics()));
}
use of io.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class LinearGutmanSplitStrategyTest method testNumChildrenSizeRoaring.
@Test
public void testNumChildrenSizeRoaring() {
BitmapFactory bf = new RoaringBitmapFactory();
RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
Random rand = new Random();
for (int i = 0; i < 100; i++) {
tree.insert(new float[] { rand.nextFloat(), rand.nextFloat() }, i);
}
Assert.assertTrue(getNumPoints(tree.getRoot()) >= tree.getSize());
}
use of io.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class LinearGutmanSplitStrategyTest method testPickSeedsRoaring.
@Test
public void testPickSeedsRoaring() throws Exception {
BitmapFactory bf = new RoaringBitmapFactory();
LinearGutmanSplitStrategy strategy = new LinearGutmanSplitStrategy(0, 50, bf);
Node node = new Node(new float[2], new float[2], true, bf);
node.addChild(new Point(new float[] { 3, 7 }, 1, bf));
node.addChild(new Point(new float[] { 1, 6 }, 1, bf));
node.addChild(new Point(new float[] { 9, 8 }, 1, bf));
node.addChild(new Point(new float[] { 2, 5 }, 1, bf));
node.addChild(new Point(new float[] { 4, 4 }, 1, bf));
node.enclose();
Node[] groups = strategy.split(node);
Assert.assertEquals(groups[0].getMinCoordinates()[0], 1.0f);
Assert.assertEquals(groups[0].getMinCoordinates()[1], 4.0f);
Assert.assertEquals(groups[1].getMinCoordinates()[0], 9.0f);
Assert.assertEquals(groups[1].getMinCoordinates()[1], 8.0f);
}
use of io.druid.collections.bitmap.RoaringBitmapFactory in project druid by druid-io.
the class DumpSegment method runBitmaps.
private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
final BitmapSerdeFactory bitmapSerdeFactory;
if (bitmapFactory instanceof ConciseBitmapFactory) {
bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
} else if (bitmapFactory instanceof RoaringBitmapFactory) {
bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
} else {
throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
}
final List<String> columnNames = getColumnsToInclude(index);
withOutputStream(new Function<OutputStream, Object>() {
@Override
public Object apply(final OutputStream out) {
try {
final JsonGenerator jg = objectMapper.getFactory().createGenerator(out);
jg.writeStartObject();
jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
jg.writeFieldName("bitmaps");
jg.writeStartObject();
for (final String columnName : columnNames) {
final Column column = index.getColumn(columnName);
final BitmapIndex bitmapIndex = column.getBitmapIndex();
if (bitmapIndex == null) {
jg.writeNullField(columnName);
} else {
jg.writeFieldName(columnName);
jg.writeStartObject();
for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
jg.writeFieldName(Strings.nullToEmpty(bitmapIndex.getValue(i)));
final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
if (decompressBitmaps) {
jg.writeStartArray();
final IntIterator iterator = bitmap.iterator();
while (iterator.hasNext()) {
final int rowNum = iterator.next();
jg.writeNumber(rowNum);
}
jg.writeEndArray();
} else {
jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap));
}
}
jg.writeEndObject();
}
}
jg.writeEndObject();
jg.writeEndObject();
jg.close();
} catch (IOException e) {
throw Throwables.propagate(e);
}
return null;
}
});
}
Aggregations