Search in sources :

Example 1 with BitmapSerdeFactory

use of org.apache.druid.segment.data.BitmapSerdeFactory in project presto by prestodb.

the class V9SegmentIndexSource method loadIndex.

@Override
public QueryableIndex loadIndex(List<ColumnHandle> columnHandles) throws IOException {
    ByteBuffer indexBuffer = ByteBuffer.wrap(segmentColumnSource.getColumnData(INDEX_METADATA_FILE_NAME));
    GenericIndexed.read(indexBuffer, STRING_STRATEGY);
    GenericIndexed<String> allDimensions = GenericIndexed.read(indexBuffer, STRING_STRATEGY);
    Interval dataInterval = Intervals.utc(indexBuffer.getLong(), indexBuffer.getLong());
    BitmapSerdeFactory segmentBitmapSerdeFactory;
    if (indexBuffer.hasRemaining()) {
        segmentBitmapSerdeFactory = JSON_MAPPER.readValue(SERIALIZER_UTILS.readString(indexBuffer), BitmapSerdeFactory.class);
    } else {
        segmentBitmapSerdeFactory = new BitmapSerde.LegacyBitmapSerdeFactory();
    }
    Metadata metadata = null;
    ByteBuffer metadataBuffer = ByteBuffer.wrap(segmentColumnSource.getColumnData(SEGMENT_METADATA_FILE_NAME));
    try {
        metadata = JSON_MAPPER.readValue(SERIALIZER_UTILS.readBytes(metadataBuffer, metadataBuffer.remaining()), Metadata.class);
    } catch (JsonParseException | JsonMappingException e) {
        // Any jackson deserialization errors are ignored e.g. if metadata contains some aggregator which
        // is no longer supported then it is OK to not use the metadata instead of failing segment loading
        log.warn(e, "Failed to load metadata for segment");
    }
    Map<String, Supplier<ColumnHolder>> columns = new HashMap<>();
    for (ColumnHandle columnHandle : columnHandles) {
        String columnName = ((DruidColumnHandle) columnHandle).getColumnName();
        columns.put(columnName, () -> createColumnHolder(columnName));
    }
    List<String> availableDimensions = Streams.stream(allDimensions.iterator()).filter(columns::containsKey).collect(toImmutableList());
    columns.put(TIME_COLUMN_NAME, () -> createColumnHolder(TIME_COLUMN_NAME));
    Indexed<String> indexed = new ListIndexed<>(availableDimensions);
    // TODO: get rid of the time column by creating Presto's SimpleQueryableIndex impl
    return new SimpleQueryableIndex(dataInterval, indexed, segmentBitmapSerdeFactory.getBitmapFactory(), columns, null, metadata, false);
}
Also used : DruidColumnHandle(com.facebook.presto.druid.DruidColumnHandle) DruidColumnHandle(com.facebook.presto.druid.DruidColumnHandle) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ListIndexed(org.apache.druid.segment.data.ListIndexed) HashMap(java.util.HashMap) BitmapSerde(org.apache.druid.segment.data.BitmapSerde) Metadata(org.apache.druid.segment.Metadata) SimpleQueryableIndex(org.apache.druid.segment.SimpleQueryableIndex) JsonParseException(com.fasterxml.jackson.core.JsonParseException) ByteBuffer(java.nio.ByteBuffer) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) Supplier(com.google.common.base.Supplier) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory) Interval(org.joda.time.Interval)

Example 2 with BitmapSerdeFactory

use of org.apache.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
                jg.writeStartObject();
                {
                    jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                    jg.writeFieldName("bitmaps");
                    jg.writeStartObject();
                    {
                        for (final String columnName : columnNames) {
                            final ColumnHolder columnHolder = index.getColumnHolder(columnName);
                            final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
                            if (bitmapIndex == null) {
                                jg.writeNullField(columnName);
                            } else {
                                jg.writeFieldName(columnName);
                                jg.writeStartObject();
                                for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                                    String val = bitmapIndex.getValue(i);
                                    // respect nulls if they are present in the dictionary
                                    jg.writeFieldName(val == null ? "null" : val);
                                    final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                                    if (decompressBitmaps) {
                                        jg.writeStartArray();
                                        final IntIterator iterator = bitmap.iterator();
                                        while (iterator.hasNext()) {
                                            final int rowNum = iterator.next();
                                            jg.writeNumber(rowNum);
                                        }
                                        jg.writeEndArray();
                                    } else {
                                        byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap);
                                        if (bytes != null) {
                                            jg.writeBinary(bytes);
                                        }
                                    }
                                }
                                jg.writeEndObject();
                            }
                        }
                    }
                    jg.writeEndObject();
                }
                jg.writeEndObject();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            return null;
        }
    });
}
Also used : ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) ColumnHolder(org.apache.druid.segment.column.ColumnHolder) IntIterator(org.roaringbitmap.IntIterator) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) Json(org.apache.druid.guice.annotations.Json) IOException(java.io.IOException) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) JsonGenerator(com.fasterxml.jackson.core.JsonGenerator) ISE(org.apache.druid.java.util.common.ISE) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory)

Example 3 with BitmapSerdeFactory

use of org.apache.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.

the class LikeFilterBenchmark method setup.

@Setup
public void setup() {
    step = (END_INT - START_INT) / cardinality;
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add((i - START_INT) / step);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}
Also used : StringBitmapIndexColumnPartSupplier(org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) Function(com.google.common.base.Function) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory) Setup(org.openjdk.jmh.annotations.Setup)

Example 4 with BitmapSerdeFactory

use of org.apache.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.

the class BoundFilterBenchmark method setup.

@Setup
public void setup() {
    step = (END_INT - START_INT) / cardinality;
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(i -> i.toString()), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add((i - START_INT) / step);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}
Also used : BenchmarkMode(org.openjdk.jmh.annotations.BenchmarkMode) Measurement(org.openjdk.jmh.annotations.Measurement) Scope(org.openjdk.jmh.annotations.Scope) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) Warmup(org.openjdk.jmh.annotations.Warmup) ArrayList(java.util.ArrayList) StringBitmapIndexColumnPartSupplier(org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier) FluentIterable(com.google.common.collect.FluentIterable) OutputTimeUnit(org.openjdk.jmh.annotations.OutputTimeUnit) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) StringComparators(org.apache.druid.query.ordering.StringComparators) GenericIndexed(org.apache.druid.segment.data.GenericIndexed) BoundDimFilter(org.apache.druid.query.filter.BoundDimFilter) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) Setup(org.openjdk.jmh.annotations.Setup) Function(com.google.common.base.Function) Mode(org.openjdk.jmh.annotations.Mode) Param(org.openjdk.jmh.annotations.Param) State(org.openjdk.jmh.annotations.State) Benchmark(org.openjdk.jmh.annotations.Benchmark) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) TimeUnit(java.util.concurrent.TimeUnit) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory) List(java.util.List) NullHandling(org.apache.druid.common.config.NullHandling) BoundFilter(org.apache.druid.segment.filter.BoundFilter) Fork(org.openjdk.jmh.annotations.Fork) Preconditions(com.google.common.base.Preconditions) ConciseSetUtils(org.apache.druid.extendedset.intset.ConciseSetUtils) BitmapIndexSelector(org.apache.druid.query.filter.BitmapIndexSelector) StringBitmapIndexColumnPartSupplier(org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) Function(com.google.common.base.Function) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory) Setup(org.openjdk.jmh.annotations.Setup)

Example 5 with BitmapSerdeFactory

use of org.apache.druid.segment.data.BitmapSerdeFactory in project druid by druid-io.

the class DimensionPredicateFilterBenchmark method setup.

@Setup
public void setup() {
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new StringBitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add(i - START_INT);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex);
}
Also used : StringBitmapIndexColumnPartSupplier(org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) Function(com.google.common.base.Function) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory) Setup(org.openjdk.jmh.annotations.Setup)

Aggregations

BitmapSerdeFactory (org.apache.druid.segment.data.BitmapSerdeFactory)8 BitmapFactory (org.apache.druid.collections.bitmap.BitmapFactory)5 RoaringBitmapSerdeFactory (org.apache.druid.segment.data.RoaringBitmapSerdeFactory)5 Function (com.google.common.base.Function)4 MutableBitmap (org.apache.druid.collections.bitmap.MutableBitmap)4 RoaringBitmapFactory (org.apache.druid.collections.bitmap.RoaringBitmapFactory)4 BitmapIndex (org.apache.druid.segment.column.BitmapIndex)4 StringBitmapIndexColumnPartSupplier (org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier)3 Setup (org.openjdk.jmh.annotations.Setup)3 Preconditions (com.google.common.base.Preconditions)2 ByteBuffer (java.nio.ByteBuffer)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 List (java.util.List)2 ImmutableBitmap (org.apache.druid.collections.bitmap.ImmutableBitmap)2 NullHandling (org.apache.druid.common.config.NullHandling)2 BitmapIndexSelector (org.apache.druid.query.filter.BitmapIndexSelector)2 DruidColumnHandle (com.facebook.presto.druid.DruidColumnHandle)1 ColumnHandle (com.facebook.presto.spi.ColumnHandle)1 JsonGenerator (com.fasterxml.jackson.core.JsonGenerator)1