use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.
the class DumpSegment method runBitmaps.
private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
final BitmapSerdeFactory bitmapSerdeFactory;
if (bitmapFactory instanceof ConciseBitmapFactory) {
bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
} else if (bitmapFactory instanceof RoaringBitmapFactory) {
bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
} else {
throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
}
final List<String> columnNames = getColumnsToInclude(index);
withOutputStream(new Function<OutputStream, Object>() {
@Override
public Object apply(final OutputStream out) {
try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
jg.writeStartObject();
{
jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
jg.writeFieldName("bitmaps");
jg.writeStartObject();
{
for (final String columnName : columnNames) {
final ColumnHolder columnHolder = index.getColumnHolder(columnName);
final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
if (bitmapIndex == null) {
jg.writeNullField(columnName);
} else {
jg.writeFieldName(columnName);
jg.writeStartObject();
for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
String val = bitmapIndex.getValue(i);
// respect nulls if they are present in the dictionary
jg.writeFieldName(val == null ? "null" : val);
final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
if (decompressBitmaps) {
jg.writeStartArray();
final IntIterator iterator = bitmap.iterator();
while (iterator.hasNext()) {
final int rowNum = iterator.next();
jg.writeNumber(rowNum);
}
jg.writeEndArray();
} else {
byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap);
if (bytes != null) {
jg.writeBinary(bytes);
}
}
}
jg.writeEndObject();
}
}
}
jg.writeEndObject();
}
jg.writeEndObject();
} catch (IOException e) {
throw new RuntimeException(e);
}
return null;
}
});
}
use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.
the class ImmutableRTreeTest method testToAndFromByteBuffer.
@Test
public void testToAndFromByteBuffer() {
BitmapFactory bf = new ConciseBitmapFactory();
RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
tree.insert(new float[] { 0, 0 }, 1);
tree.insert(new float[] { 1, 1 }, 2);
tree.insert(new float[] { 2, 2 }, 3);
tree.insert(new float[] { 3, 3 }, 4);
tree.insert(new float[] { 4, 4 }, 5);
ImmutableRTree firstTree = ImmutableRTree.newImmutableFromMutable(tree);
ByteBuffer buffer = ByteBuffer.wrap(firstTree.toBytes());
ImmutableRTree secondTree = new ImmutableRTree(buffer, bf);
Iterable<ImmutableBitmap> points = secondTree.search(new RadiusBound(new float[] { 0, 0 }, 10));
ImmutableBitmap finalSet = bf.union(points);
Assert.assertTrue(finalSet.size() >= 5);
Set<Integer> expected = Sets.newHashSet(1, 2, 3, 4, 5);
IntIterator iter = finalSet.iterator();
while (iter.hasNext()) {
Assert.assertTrue(expected.contains(iter.next()));
}
}
use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.
the class ImmutableRTreeTest method testSearchWithSplit2.
@Test
public void testSearchWithSplit2() {
BitmapFactory bf = new ConciseBitmapFactory();
RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
tree.insert(new float[] { 0.0f, 0.0f }, 0);
tree.insert(new float[] { 1.0f, 3.0f }, 1);
tree.insert(new float[] { 4.0f, 2.0f }, 2);
tree.insert(new float[] { 7.0f, 3.0f }, 3);
tree.insert(new float[] { 8.0f, 6.0f }, 4);
Random rand = ThreadLocalRandom.current();
for (int i = 5; i < 5000; i++) {
tree.insert(new float[] { (float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0) }, i);
}
ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
Iterable<ImmutableBitmap> points = searchTree.search(new RectangularBound(new float[] { 0, 0 }, new float[] { 9, 9 }));
ImmutableBitmap finalSet = bf.union(points);
Assert.assertTrue(finalSet.size() >= 5);
Set<Integer> expected = Sets.newHashSet(0, 1, 2, 3, 4);
IntIterator iter = finalSet.iterator();
while (iter.hasNext()) {
Assert.assertTrue(expected.contains(iter.next()));
}
}
use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.
the class ImmutableRTreeTest method showBenchmarks.
// TODO rewrite to JMH and move to the benchmarks project
@SuppressWarnings("unused")
public void showBenchmarks() {
final int start = 1;
final int factor = 10;
final int end = 10000000;
final int radius = 10;
for (int numPoints = start; numPoints <= end; numPoints *= factor) {
try {
BitmapFactory bf = new ConciseBitmapFactory();
RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
Stopwatch stopwatch = Stopwatch.createStarted();
Random rand = ThreadLocalRandom.current();
for (int i = 0; i < numPoints; i++) {
tree.insert(new float[] { (float) (rand.nextDouble() * 100), (float) (rand.nextDouble() * 100) }, i);
}
long stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
System.out.printf(Locale.ENGLISH, "[%,d]: insert = %,d ms%n", numPoints, stop);
stopwatch.reset().start();
ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
System.out.printf(Locale.ENGLISH, "[%,d]: size = %,d bytes%n", numPoints, searchTree.toBytes().length);
System.out.printf(Locale.ENGLISH, "[%,d]: buildImmutable = %,d ms%n", numPoints, stop);
stopwatch.reset().start();
Iterable<ImmutableBitmap> points = searchTree.search(new RadiusBound(new float[] { 50, 50 }, radius));
Iterables.size(points);
stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
System.out.printf(Locale.ENGLISH, "[%,d]: search = %,dms%n", numPoints, stop);
stopwatch.reset().start();
ImmutableBitmap finalSet = bf.union(points);
stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
System.out.printf(Locale.ENGLISH, "[%,d]: union of %,d points in %,d ms%n", numPoints, finalSet.size(), stop);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.
the class RTreeTest method setUp.
@Before
public void setUp() {
BitmapFactory bf = new ConciseBitmapFactory();
tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
BitmapFactory rbf = new RoaringBitmapFactory();
roaringtree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, rbf), rbf);
}
Aggregations