Search in sources :

Example 1 with ConciseBitmapSerdeFactory

use of org.apache.druid.segment.data.ConciseBitmapSerdeFactory in project druid by druid-io.

the class NewestSegmentFirstPolicyTest method testIteratorReturnsSegmentsAsCompactionStateChangedWithCompactedStateHasSameSegmentGranularity.

@Test
public void testIteratorReturnsSegmentsAsCompactionStateChangedWithCompactedStateHasSameSegmentGranularity() {
    // Different indexSpec as what is set in the auto compaction config
    IndexSpec newIndexSpec = new IndexSpec(new ConciseBitmapSerdeFactory(), null, null, null);
    Map<String, Object> newIndexSpecMap = mapper.convertValue(newIndexSpec, new TypeReference<Map<String, Object>>() {
    });
    PartitionsSpec partitionsSpec = NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(null, null));
    // Create segments that were compacted (CompactionState != null) and have segmentGranularity=DAY
    final VersionedIntervalTimeline<String, DataSegment> timeline = createTimeline(new SegmentGenerateSpec(Intervals.of("2017-10-02T00:00:00/2017-10-03T00:00:00"), new Period("P1D"), null, new CompactionState(partitionsSpec, null, null, null, newIndexSpecMap, null)));
    // Duration of new segmentGranularity is the same as before (P1D)
    final CompactionSegmentIterator iterator = policy.reset(ImmutableMap.of(DATA_SOURCE, createCompactionConfig(130000, new Period("P0D"), new UserCompactionTaskGranularityConfig(new PeriodGranularity(new Period("P1D"), null, DateTimeZone.UTC), null, null))), ImmutableMap.of(DATA_SOURCE, timeline), Collections.emptyMap());
    // We should get all segments in timeline back since indexSpec changed
    Assert.assertTrue(iterator.hasNext());
    List<DataSegment> expectedSegmentsToCompact = new ArrayList<>(timeline.findNonOvershadowedObjectsInInterval(Intervals.of("2017-10-01T00:00:00/2017-10-03T00:00:00"), Partitions.ONLY_COMPLETE));
    Assert.assertEquals(ImmutableSet.copyOf(expectedSegmentsToCompact), ImmutableSet.copyOf(iterator.next()));
    // No more
    Assert.assertFalse(iterator.hasNext());
}
Also used : IndexSpec(org.apache.druid.segment.IndexSpec) PeriodGranularity(org.apache.druid.java.util.common.granularity.PeriodGranularity) ArrayList(java.util.ArrayList) Period(org.joda.time.Period) DataSegment(org.apache.druid.timeline.DataSegment) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) PartitionsSpec(org.apache.druid.indexer.partitions.PartitionsSpec) CompactionState(org.apache.druid.timeline.CompactionState) UserCompactionTaskGranularityConfig(org.apache.druid.server.coordinator.UserCompactionTaskGranularityConfig) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) Test(org.junit.Test)

Example 2 with ConciseBitmapSerdeFactory

use of org.apache.druid.segment.data.ConciseBitmapSerdeFactory in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
                jg.writeStartObject();
                {
                    jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                    jg.writeFieldName("bitmaps");
                    jg.writeStartObject();
                    {
                        for (final String columnName : columnNames) {
                            final ColumnHolder columnHolder = index.getColumnHolder(columnName);
                            final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
                            if (bitmapIndex == null) {
                                jg.writeNullField(columnName);
                            } else {
                                jg.writeFieldName(columnName);
                                jg.writeStartObject();
                                for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                                    String val = bitmapIndex.getValue(i);
                                    // respect nulls if they are present in the dictionary
                                    jg.writeFieldName(val == null ? "null" : val);
                                    final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                                    if (decompressBitmaps) {
                                        jg.writeStartArray();
                                        final IntIterator iterator = bitmap.iterator();
                                        while (iterator.hasNext()) {
                                            final int rowNum = iterator.next();
                                            jg.writeNumber(rowNum);
                                        }
                                        jg.writeEndArray();
                                    } else {
                                        byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap);
                                        if (bytes != null) {
                                            jg.writeBinary(bytes);
                                        }
                                    }
                                }
                                jg.writeEndObject();
                            }
                        }
                    }
                    jg.writeEndObject();
                }
                jg.writeEndObject();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            return null;
        }
    });
}
Also used : ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) ColumnHolder(org.apache.druid.segment.column.ColumnHolder) IntIterator(org.roaringbitmap.IntIterator) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) BitmapIndex(org.apache.druid.segment.column.BitmapIndex) Json(org.apache.druid.guice.annotations.Json) IOException(java.io.IOException) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) JsonGenerator(com.fasterxml.jackson.core.JsonGenerator) ISE(org.apache.druid.java.util.common.ISE) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory)

Example 3 with ConciseBitmapSerdeFactory

use of org.apache.druid.segment.data.ConciseBitmapSerdeFactory in project druid by druid-io.

the class DictionaryEncodedColumnPartSerdeTest method testSerde.

@Test
public void testSerde() throws Exception {
    // bitmapSerdeFactory not specified
    String json = "{\n" + " \"type\": \"stringDictionary\",\n" + " \"byteOrder\": \"BIG_ENDIAN\"\n" + "}\n";
    ObjectMapper mapper = TestHelper.makeJsonMapper();
    DictionaryEncodedColumnPartSerde serde = (DictionaryEncodedColumnPartSerde) mapper.readValue(mapper.writeValueAsString(mapper.readValue(json, ColumnPartSerde.class)), ColumnPartSerde.class);
    Assert.assertEquals(ByteOrder.BIG_ENDIAN, serde.getByteOrder());
    Assert.assertTrue(serde.getBitmapSerdeFactory() instanceof ConciseBitmapSerdeFactory);
    // bitmapSerdeFactory specified
    json = "{\n" + "\"type\": \"stringDictionary\",\n" + "\"byteOrder\": \"LITTLE_ENDIAN\",\n" + "\"bitmapSerdeFactory\": { \"type\": \"roaring\" }\n" + "}";
    serde = (DictionaryEncodedColumnPartSerde) mapper.readValue(mapper.writeValueAsString(mapper.readValue(json, ColumnPartSerde.class)), ColumnPartSerde.class);
    Assert.assertEquals(ByteOrder.LITTLE_ENDIAN, serde.getByteOrder());
    Assert.assertTrue(serde.getBitmapSerdeFactory() instanceof RoaringBitmapSerdeFactory);
}
Also used : RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Test(org.junit.Test)

Example 4 with ConciseBitmapSerdeFactory

use of org.apache.druid.segment.data.ConciseBitmapSerdeFactory in project druid by druid-io.

the class BaseFilterTest method makeConstructors.

public static Collection<Object[]> makeConstructors() {
    final List<Object[]> constructors = new ArrayList<>();
    final Map<String, BitmapSerdeFactory> bitmapSerdeFactories = ImmutableMap.of("concise", new ConciseBitmapSerdeFactory(), "roaring", new RoaringBitmapSerdeFactory(true));
    final Map<String, SegmentWriteOutMediumFactory> segmentWriteOutMediumFactories = ImmutableMap.of("tmpFile segment write-out medium", TmpFileSegmentWriteOutMediumFactory.instance(), "off-heap memory segment write-out medium", OffHeapMemorySegmentWriteOutMediumFactory.instance());
    final Map<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finishers = ImmutableMap.<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>>builder().put("incremental", input -> {
        final IncrementalIndex index = input.buildIncrementalIndex();
        return Pair.of(new IncrementalIndexStorageAdapter(index), index);
    }).put("mmapped", input -> {
        final QueryableIndex index = input.buildMMappedIndex();
        return Pair.of(new QueryableIndexStorageAdapter(index), index);
    }).put("mmappedMerged", input -> {
        final QueryableIndex index = input.buildMMappedMergedIndex();
        return Pair.of(new QueryableIndexStorageAdapter(index), index);
    }).put("rowBasedWithoutTypeSignature", input -> Pair.of(input.buildRowBasedSegmentWithoutTypeSignature().asStorageAdapter(), () -> {
    })).put("rowBasedWithTypeSignature", input -> Pair.of(input.buildRowBasedSegmentWithTypeSignature().asStorageAdapter(), () -> {
    })).build();
    for (Map.Entry<String, BitmapSerdeFactory> bitmapSerdeFactoryEntry : bitmapSerdeFactories.entrySet()) {
        for (Map.Entry<String, SegmentWriteOutMediumFactory> segmentWriteOutMediumFactoryEntry : segmentWriteOutMediumFactories.entrySet()) {
            for (Map.Entry<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finisherEntry : finishers.entrySet()) {
                for (boolean cnf : ImmutableList.of(false, true)) {
                    for (boolean optimize : ImmutableList.of(false, true)) {
                        final String testName = StringUtils.format("bitmaps[%s], indexMerger[%s], finisher[%s], cnf[%s], optimize[%s]", bitmapSerdeFactoryEntry.getKey(), segmentWriteOutMediumFactoryEntry.getKey(), finisherEntry.getKey(), cnf, optimize);
                        final IndexBuilder indexBuilder = IndexBuilder.create().schema(DEFAULT_INDEX_SCHEMA).indexSpec(new IndexSpec(bitmapSerdeFactoryEntry.getValue(), null, null, null)).segmentWriteOutMediumFactory(segmentWriteOutMediumFactoryEntry.getValue());
                        constructors.add(new Object[] { testName, indexBuilder, finisherEntry.getValue(), cnf, optimize });
                    }
                }
            }
        }
    }
    return constructors;
}
Also used : Arrays(java.util.Arrays) LongDimensionSchema(org.apache.druid.data.input.impl.LongDimensionSchema) RowAdapters(org.apache.druid.segment.RowAdapters) IndexSpec(org.apache.druid.segment.IndexSpec) ExprType(org.apache.druid.math.expr.ExprType) TimestampSpec(org.apache.druid.data.input.impl.TimestampSpec) IndexedInts(org.apache.druid.segment.data.IndexedInts) StorageAdapter(org.apache.druid.segment.StorageAdapter) TmpFileSegmentWriteOutMediumFactory(org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory) ByteBuffer(java.nio.ByteBuffer) Pair(org.apache.druid.java.util.common.Pair) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) ColumnSelectorFactory(org.apache.druid.segment.ColumnSelectorFactory) IncrementalIndexStorageAdapter(org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter) ExpressionType(org.apache.druid.math.expr.ExpressionType) Expr(org.apache.druid.math.expr.Expr) Map(java.util.Map) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) OffHeapMemorySegmentWriteOutMediumFactory(org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory) Parameterized(org.junit.runners.Parameterized) DateTimes(org.apache.druid.java.util.common.DateTimes) Sequence(org.apache.druid.java.util.common.guava.Sequence) RowBasedColumnSelectorFactory(org.apache.druid.segment.RowBasedColumnSelectorFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) Function(com.google.common.base.Function) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) QueryableIndex(org.apache.druid.segment.QueryableIndex) StringUtils(org.apache.druid.java.util.common.StringUtils) Set(java.util.Set) ISE(org.apache.druid.java.util.common.ISE) IndexBuilder(org.apache.druid.segment.IndexBuilder) VectorObjectSelector(org.apache.druid.segment.vector.VectorObjectSelector) InputRowParser(org.apache.druid.data.input.impl.InputRowParser) TestExprMacroTable(org.apache.druid.query.expression.TestExprMacroTable) VectorValueSelector(org.apache.druid.segment.vector.VectorValueSelector) VectorColumnSelectorFactory(org.apache.druid.segment.vector.VectorColumnSelectorFactory) IncrementalIndexSchema(org.apache.druid.segment.incremental.IncrementalIndexSchema) InputRow(org.apache.druid.data.input.InputRow) List(java.util.List) DimensionSchema(org.apache.druid.data.input.impl.DimensionSchema) DimFilter(org.apache.druid.query.filter.DimFilter) BitmapIndexSelector(org.apache.druid.query.filter.BitmapIndexSelector) Iterables(com.google.common.collect.Iterables) DoubleDimensionSchema(org.apache.druid.data.input.impl.DoubleDimensionSchema) Intervals(org.apache.druid.java.util.common.Intervals) FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) Parser(org.apache.druid.math.expr.Parser) SegmentWriteOutMediumFactory(org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory) HashMap(java.util.HashMap) TimeAndDimsParseSpec(org.apache.druid.data.input.impl.TimeAndDimsParseSpec) ArrayList(java.util.ArrayList) VectorCursor(org.apache.druid.segment.vector.VectorCursor) ImmutableList(com.google.common.collect.ImmutableList) FloatDimensionSchema(org.apache.druid.data.input.impl.FloatDimensionSchema) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) SettableSupplier(org.apache.druid.common.guava.SettableSupplier) VectorValueMatcher(org.apache.druid.query.filter.vector.VectorValueMatcher) RowBasedStorageAdapter(org.apache.druid.segment.RowBasedStorageAdapter) DimensionSelector(org.apache.druid.segment.DimensionSelector) VectorAggregator(org.apache.druid.query.aggregation.VectorAggregator) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) Sequences(org.apache.druid.java.util.common.guava.Sequences) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) SingleValueDimensionVectorSelector(org.apache.druid.segment.vector.SingleValueDimensionVectorSelector) Nullable(javax.annotation.Nullable) Before(org.junit.Before) ValueMatcher(org.apache.druid.query.filter.ValueMatcher) BitmapResultFactory(org.apache.druid.query.BitmapResultFactory) ColumnInspector(org.apache.druid.segment.ColumnInspector) QueryableIndexStorageAdapter(org.apache.druid.segment.QueryableIndexStorageAdapter) VirtualColumns(org.apache.druid.segment.VirtualColumns) MapInputRowParser(org.apache.druid.data.input.impl.MapInputRowParser) DimensionsSpec(org.apache.druid.data.input.impl.DimensionsSpec) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Aggregator(org.apache.druid.query.aggregation.Aggregator) Maps(com.google.common.collect.Maps) ColumnSelector(org.apache.druid.segment.ColumnSelector) Granularities(org.apache.druid.java.util.common.granularity.Granularities) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory) Rule(org.junit.Rule) Cursor(org.apache.druid.segment.Cursor) NullHandling(org.apache.druid.common.config.NullHandling) RowSignature(org.apache.druid.segment.column.RowSignature) Closeable(java.io.Closeable) ColumnType(org.apache.druid.segment.column.ColumnType) Preconditions(com.google.common.base.Preconditions) Assert(org.junit.Assert) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) Filter(org.apache.druid.query.filter.Filter) IndexSpec(org.apache.druid.segment.IndexSpec) Closeable(java.io.Closeable) ArrayList(java.util.ArrayList) StorageAdapter(org.apache.druid.segment.StorageAdapter) IncrementalIndexStorageAdapter(org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter) RowBasedStorageAdapter(org.apache.druid.segment.RowBasedStorageAdapter) QueryableIndexStorageAdapter(org.apache.druid.segment.QueryableIndexStorageAdapter) TmpFileSegmentWriteOutMediumFactory(org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory) OffHeapMemorySegmentWriteOutMediumFactory(org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory) SegmentWriteOutMediumFactory(org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory) Function(com.google.common.base.Function) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) QueryableIndexStorageAdapter(org.apache.druid.segment.QueryableIndexStorageAdapter) IndexBuilder(org.apache.druid.segment.IndexBuilder) QueryableIndex(org.apache.druid.segment.QueryableIndex) IncrementalIndexStorageAdapter(org.apache.druid.segment.incremental.IncrementalIndexStorageAdapter) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) ConciseBitmapSerdeFactory(org.apache.druid.segment.data.ConciseBitmapSerdeFactory) RoaringBitmapSerdeFactory(org.apache.druid.segment.data.RoaringBitmapSerdeFactory) BitmapSerdeFactory(org.apache.druid.segment.data.BitmapSerdeFactory)

Aggregations

ConciseBitmapSerdeFactory (org.apache.druid.segment.data.ConciseBitmapSerdeFactory)4 RoaringBitmapSerdeFactory (org.apache.druid.segment.data.RoaringBitmapSerdeFactory)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ArrayList (java.util.ArrayList)2 Map (java.util.Map)2 ISE (org.apache.druid.java.util.common.ISE)2 JsonGenerator (com.fasterxml.jackson.core.JsonGenerator)1 Function (com.google.common.base.Function)1 Preconditions (com.google.common.base.Preconditions)1 ImmutableList (com.google.common.collect.ImmutableList)1 Iterables (com.google.common.collect.Iterables)1 Maps (com.google.common.collect.Maps)1 Closeable (java.io.Closeable)1 FileOutputStream (java.io.FileOutputStream)1 IOException (java.io.IOException)1 OutputStream (java.io.OutputStream)1 ByteBuffer (java.nio.ByteBuffer)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1