Examples with BitmapSerdeFactory io.druid.segment.data.BitmapSerdeFactory used on opensource projects

Example 6 with BitmapSerdeFactory

use of io.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.

the class StringDimensionMergerV9 method writeIndexes.

@Override
public void writeIndexes(List<IntBuffer> segmentRowNumConversions, Closer closer) throws IOException {
    long dimStartTime = System.currentTimeMillis();
    final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
    String bmpFilename = String.format("%s.inverted", dimensionName);
    bitmapWriter = new GenericIndexedWriter<>(ioPeon, bmpFilename, bitmapSerdeFactory.getObjectStrategy());
    bitmapWriter.open();
    // write dim values to one single file because we need to read it
    File dimValueFile = IndexIO.makeDimFile(outDir, dimensionName);
    try (FileOutputStream fos = new FileOutputStream(dimValueFile)) {
        ByteStreams.copy(dictionaryWriter.combineStreams(), fos);
    }
    final MappedByteBuffer dimValsMapped = Files.map(dimValueFile);
    try (Closeable toCloseEncodedValueWriter = encodedValueWriter;
        Closeable toCloseBitmapWriter = bitmapWriter;
        Closeable dimValsMappedUnmapper = new Closeable() {

            @Override
            public void close() {
                ByteBufferUtils.unmap(dimValsMapped);
            }
        }) {
        Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY);
        BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory();
        RTree tree = null;
        boolean hasSpatial = capabilities.hasSpatialIndexes();
        if (hasSpatial) {
            spatialWriter = new ByteBufferWriter<>(ioPeon, String.format("%s.spatial", dimensionName), new IndexedRTree.ImmutableRTreeObjectStrategy(bmpFactory));
            spatialWriter.open();
            tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bmpFactory), bmpFactory);
        }
        IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName);
        //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result.
        for (int dictId = 0; dictId < dimVals.size(); dictId++) {
            progress.progress();
            mergeBitmaps(segmentRowNumConversions, dimVals, bmpFactory, tree, hasSpatial, dictIdSeeker, dictId, adapters, dimensionName, nullRowsBitmap, bitmapWriter);
        }
        if (hasSpatial) {
            spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
            spatialWriter.close();
        }
        log.info("Completed dim[%s] inverted with cardinality[%,d] in %,d millis.", dimensionName, dimVals.size(), System.currentTimeMillis() - dimStartTime);
    }
}

Example 7 with BitmapSerdeFactory

use of io.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try {
                final JsonGenerator jg = objectMapper.getFactory().createGenerator(out);
                jg.writeStartObject();
                jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                jg.writeFieldName("bitmaps");
                jg.writeStartObject();
                for (final String columnName : columnNames) {
                    final Column column = index.getColumn(columnName);
                    final BitmapIndex bitmapIndex = column.getBitmapIndex();
                    if (bitmapIndex == null) {
                        jg.writeNullField(columnName);
                    } else {
                        jg.writeFieldName(columnName);
                        jg.writeStartObject();
                        for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                            jg.writeFieldName(Strings.nullToEmpty(bitmapIndex.getValue(i)));
                            final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                            if (decompressBitmaps) {
                                jg.writeStartArray();
                                final IntIterator iterator = bitmap.iterator();
                                while (iterator.hasNext()) {
                                    final int rowNum = iterator.next();
                                    jg.writeNumber(rowNum);
                                }
                                jg.writeEndArray();
                            } else {
                                jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap));
                            }
                        }
                        jg.writeEndObject();
                    }
                }
                jg.writeEndObject();
                jg.writeEndObject();
                jg.close();
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
            return null;
        }
    });
}

Example 8 with BitmapSerdeFactory

use of io.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.

the class BaseFilterTest method makeConstructors.

public static Collection<Object[]> makeConstructors() {
    final List<Object[]> constructors = Lists.newArrayList();
    final Map<String, BitmapSerdeFactory> bitmapSerdeFactories = ImmutableMap.<String, BitmapSerdeFactory>of("concise", new ConciseBitmapSerdeFactory(), "roaring", new RoaringBitmapSerdeFactory(true));
    final Map<String, IndexMerger> indexMergers = ImmutableMap.<String, IndexMerger>of("IndexMerger", TestHelper.getTestIndexMerger(), "IndexMergerV9", TestHelper.getTestIndexMergerV9());
    final Map<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finishers = ImmutableMap.of("incremental", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {

        @Override
        public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
            final IncrementalIndex index = input.buildIncrementalIndex();
            return Pair.<StorageAdapter, Closeable>of(new IncrementalIndexStorageAdapter(index), new Closeable() {

                @Override
                public void close() throws IOException {
                    index.close();
                }
            });
        }
    }, "mmapped", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {

        @Override
        public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
            final QueryableIndex index = input.buildMMappedIndex();
            return Pair.<StorageAdapter, Closeable>of(new QueryableIndexStorageAdapter(index), new Closeable() {

                @Override
                public void close() throws IOException {
                    index.close();
                }
            });
        }
    }, "mmappedMerged", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {

        @Override
        public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
            final QueryableIndex index = input.buildMMappedMergedIndex();
            return Pair.<StorageAdapter, Closeable>of(new QueryableIndexStorageAdapter(index), new Closeable() {

                @Override
                public void close() throws IOException {
                    index.close();
                }
            });
        }
    });
    for (Map.Entry<String, BitmapSerdeFactory> bitmapSerdeFactoryEntry : bitmapSerdeFactories.entrySet()) {
        for (Map.Entry<String, IndexMerger> indexMergerEntry : indexMergers.entrySet()) {
            for (Map.Entry<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finisherEntry : finishers.entrySet()) {
                for (boolean cnf : ImmutableList.of(false, true)) {
                    for (boolean optimize : ImmutableList.of(false, true)) {
                        final String testName = String.format("bitmaps[%s], indexMerger[%s], finisher[%s], optimize[%s]", bitmapSerdeFactoryEntry.getKey(), indexMergerEntry.getKey(), finisherEntry.getKey(), optimize);
                        final IndexBuilder indexBuilder = IndexBuilder.create().indexSpec(new IndexSpec(bitmapSerdeFactoryEntry.getValue(), null, null, null)).indexMerger(indexMergerEntry.getValue());
                        constructors.add(new Object[] { testName, indexBuilder, finisherEntry.getValue(), cnf, optimize });
                    }
                }
            }
        }
    }
    return constructors;
}