Examples with BitmapFactory io.druid.collections.bitmap.BitmapFactory used on opensource projects

Example 11 with BitmapFactory

use of io.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class LinearGutmanSplitStrategyTest method testPickSeedsRoaring.

@Test
public void testPickSeedsRoaring() throws Exception {
    BitmapFactory bf = new RoaringBitmapFactory();
    LinearGutmanSplitStrategy strategy = new LinearGutmanSplitStrategy(0, 50, bf);
    Node node = new Node(new float[2], new float[2], true, bf);
    node.addChild(new Point(new float[] { 3, 7 }, 1, bf));
    node.addChild(new Point(new float[] { 1, 6 }, 1, bf));
    node.addChild(new Point(new float[] { 9, 8 }, 1, bf));
    node.addChild(new Point(new float[] { 2, 5 }, 1, bf));
    node.addChild(new Point(new float[] { 4, 4 }, 1, bf));
    node.enclose();
    Node[] groups = strategy.split(node);
    Assert.assertEquals(groups[0].getMinCoordinates()[0], 1.0f);
    Assert.assertEquals(groups[0].getMinCoordinates()[1], 4.0f);
    Assert.assertEquals(groups[1].getMinCoordinates()[0], 9.0f);
    Assert.assertEquals(groups[1].getMinCoordinates()[1], 8.0f);
}

Example 12 with BitmapFactory

use of io.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class StringDimensionMergerV9 method writeIndexes.

@Override
public void writeIndexes(List<IntBuffer> segmentRowNumConversions, Closer closer) throws IOException {
    long dimStartTime = System.currentTimeMillis();
    final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
    String bmpFilename = String.format("%s.inverted", dimensionName);
    bitmapWriter = new GenericIndexedWriter<>(ioPeon, bmpFilename, bitmapSerdeFactory.getObjectStrategy());
    bitmapWriter.open();
    // write dim values to one single file because we need to read it
    File dimValueFile = IndexIO.makeDimFile(outDir, dimensionName);
    try (FileOutputStream fos = new FileOutputStream(dimValueFile)) {
        ByteStreams.copy(dictionaryWriter.combineStreams(), fos);
    }
    final MappedByteBuffer dimValsMapped = Files.map(dimValueFile);
    try (Closeable toCloseEncodedValueWriter = encodedValueWriter;
        Closeable toCloseBitmapWriter = bitmapWriter;
        Closeable dimValsMappedUnmapper = new Closeable() {

            @Override
            public void close() {
                ByteBufferUtils.unmap(dimValsMapped);
            }
        }) {
        Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY);
        BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory();
        RTree tree = null;
        boolean hasSpatial = capabilities.hasSpatialIndexes();
        if (hasSpatial) {
            spatialWriter = new ByteBufferWriter<>(ioPeon, String.format("%s.spatial", dimensionName), new IndexedRTree.ImmutableRTreeObjectStrategy(bmpFactory));
            spatialWriter.open();
            tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bmpFactory), bmpFactory);
        }
        IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName);
        //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result.
        for (int dictId = 0; dictId < dimVals.size(); dictId++) {
            progress.progress();
            mergeBitmaps(segmentRowNumConversions, dimVals, bmpFactory, tree, hasSpatial, dictIdSeeker, dictId, adapters, dimensionName, nullRowsBitmap, bitmapWriter);
        }
        if (hasSpatial) {
            spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
            spatialWriter.close();
        }
        log.info("Completed dim[%s] inverted with cardinality[%,d] in %,d millis.", dimensionName, dimVals.size(), System.currentTimeMillis() - dimStartTime);
    }
}

Example 13 with BitmapFactory

use of io.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try {
                final JsonGenerator jg = objectMapper.getFactory().createGenerator(out);
                jg.writeStartObject();
                jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                jg.writeFieldName("bitmaps");
                jg.writeStartObject();
                for (final String columnName : columnNames) {
                    final Column column = index.getColumn(columnName);
                    final BitmapIndex bitmapIndex = column.getBitmapIndex();
                    if (bitmapIndex == null) {
                        jg.writeNullField(columnName);
                    } else {
                        jg.writeFieldName(columnName);
                        jg.writeStartObject();
                        for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                            jg.writeFieldName(Strings.nullToEmpty(bitmapIndex.getValue(i)));
                            final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                            if (decompressBitmaps) {
                                jg.writeStartArray();
                                final IntIterator iterator = bitmap.iterator();
                                while (iterator.hasNext()) {
                                    final int rowNum = iterator.next();
                                    jg.writeNumber(rowNum);
                                }
                                jg.writeEndArray();
                            } else {
                                jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap));
                            }
                        }
                        jg.writeEndObject();
                    }
                }
                jg.writeEndObject();
                jg.writeEndObject();
                jg.close();
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
            return null;
        }
    });
}

Example 14 with BitmapFactory

use of io.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class FiltersTest method makeNonOverlappedBitmapIndexes.

private static BitmapIndex makeNonOverlappedBitmapIndexes(final int bitmapNum, final List<ImmutableBitmap> bitmaps) {
    final BitmapIndex bitmapIndex = getBitmapIndex(bitmaps);
    final BitmapFactory factory = bitmapIndex.getBitmapFactory();
    for (int i = 0; i < bitmapNum; i++) {
        final MutableBitmap mutableBitmap = factory.makeEmptyMutableBitmap();
        for (int j = 0; j < 10; j++) {
            mutableBitmap.add(i * 10 + j);
        }
        bitmaps.add(factory.makeImmutableBitmap(mutableBitmap));
    }
    return bitmapIndex;
}

Example 15 with BitmapFactory

use of io.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ImmutableRTreeTest method showBenchmarks.

//@Test
public void showBenchmarks() {
    final int start = 1;
    final int factor = 10;
    final int end = 10000000;
    final int radius = 10;
    for (int numPoints = start; numPoints <= end; numPoints *= factor) {
        try {
            BitmapFactory bf = new ConciseBitmapFactory();
            RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
            Stopwatch stopwatch = Stopwatch.createStarted();
            Random rand = new Random();
            for (int i = 0; i < numPoints; i++) {
                tree.insert(new float[] { (float) (rand.nextDouble() * 100), (float) (rand.nextDouble() * 100) }, i);
            }
            long stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            System.out.printf("[%,d]: insert = %,d ms%n", numPoints, stop);
            stopwatch.reset().start();
            ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
            stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            System.out.printf("[%,d]: size = %,d bytes%n", numPoints, searchTree.toBytes().length);
            System.out.printf("[%,d]: buildImmutable = %,d ms%n", numPoints, stop);
            stopwatch.reset().start();
            Iterable<ImmutableBitmap> points = searchTree.search(new RadiusBound(new float[] { 50, 50 }, radius));
            Iterables.size(points);
            stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            System.out.printf("[%,d]: search = %,dms%n", numPoints, stop);
            stopwatch.reset().start();
            ImmutableBitmap finalSet = bf.union(points);
            stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            System.out.printf("[%,d]: union of %,d points in %,d ms%n", numPoints, finalSet.size(), stop);
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    }
}