Examples with RoaringBitmapSerdeFactory org.apache.druid.segment.data.RoaringBitmapSerdeFactory used on opensource projects

Example 1 with RoaringBitmapSerdeFactory

use of org.apache.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try (final JsonGenerator jg = objectMapper.getFactory().createGenerator(out)) {
                jg.writeStartObject();
                {
                    jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                    jg.writeFieldName("bitmaps");
                    jg.writeStartObject();
                    {
                        for (final String columnName : columnNames) {
                            final ColumnHolder columnHolder = index.getColumnHolder(columnName);
                            final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
                            if (bitmapIndex == null) {
                                jg.writeNullField(columnName);
                            } else {
                                jg.writeFieldName(columnName);
                                jg.writeStartObject();
                                for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                                    String val = bitmapIndex.getValue(i);
                                    // respect nulls if they are present in the dictionary
                                    jg.writeFieldName(val == null ? "null" : val);
                                    final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                                    if (decompressBitmaps) {
                                        jg.writeStartArray();
                                        final IntIterator iterator = bitmap.iterator();
                                        while (iterator.hasNext()) {
                                            final int rowNum = iterator.next();
                                            jg.writeNumber(rowNum);
                                        }
                                        jg.writeEndArray();
                                    } else {
                                        byte[] bytes = bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap);
                                        if (bytes != null) {
                                            jg.writeBinary(bytes);
                                        }
                                    }
                                }
                                jg.writeEndObject();
                            }
                        }
                    }
                    jg.writeEndObject();
                }
                jg.writeEndObject();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            return null;
        }
    });
}

Example 2 with RoaringBitmapSerdeFactory

use of org.apache.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class CompactionTaskTest method testGetTuningConfigWithParallelIndexTuningConfig.

@Test
public void testGetTuningConfigWithParallelIndexTuningConfig() {
    ParallelIndexTuningConfig parallelIndexTuningConfig = new ParallelIndexTuningConfig(null, // null to compute maxRowsPerSegment automatically
    null, null, 500000, 1000000L, null, null, null, null, null, new IndexSpec(new RoaringBitmapSerdeFactory(true), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), null, null, true, false, 5000L, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
    CompactionTask.CompactionTuningConfig compactionTuningConfig = new CompactionTask.CompactionTuningConfig(null, // null to compute maxRowsPerSegment automatically
    null, null, 500000, 1000000L, null, null, null, null, null, new IndexSpec(new RoaringBitmapSerdeFactory(true), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), null, null, true, false, 5000L, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
    Assert.assertEquals(compactionTuningConfig, CompactionTask.getTuningConfig(parallelIndexTuningConfig));
}

Example 3 with RoaringBitmapSerdeFactory

use of org.apache.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class IndexTaskSerdeTest method testSerdeTuningConfigWithDynamicPartitionsSpec.

@Test
public void testSerdeTuningConfigWithDynamicPartitionsSpec() throws IOException {
    final IndexTuningConfig tuningConfig = new IndexTuningConfig(null, null, null, 100, 2000L, null, null, null, null, null, new DynamicPartitionsSpec(1000, 2000L), new IndexSpec(new RoaringBitmapSerdeFactory(false), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), null, null, false, null, null, 100L, OffHeapMemorySegmentWriteOutMediumFactory.instance(), true, 10, 100, 1234, 0L);
    assertSerdeTuningConfig(tuningConfig);
}

Example 4 with RoaringBitmapSerdeFactory

use of org.apache.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class IndexTaskSerdeTest method testSerdeTuningConfigWithDeprecatedHashedPartitionsSpec.

@Test
public void testSerdeTuningConfigWithDeprecatedHashedPartitionsSpec() throws IOException {
    final IndexTuningConfig tuningConfig = new IndexTuningConfig(null, null, null, 100, 2000L, null, null, null, 10, ImmutableList.of("dim1", "dim2"), null, new IndexSpec(new RoaringBitmapSerdeFactory(false), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), null, null, false, null, null, 100L, OffHeapMemorySegmentWriteOutMediumFactory.instance(), true, 10, 100, 1234, null);
    assertSerdeTuningConfig(tuningConfig);
}

Example 5 with RoaringBitmapSerdeFactory

use of org.apache.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class IndexTaskSerdeTest method testForceGuaranteedRollupWithDynamicPartitionsSpec.

@Test
public void testForceGuaranteedRollupWithDynamicPartitionsSpec() {
    expectedException.expect(IllegalArgumentException.class);
    expectedException.expectMessage("DynamicPartitionsSpec cannot be used for perfect rollup");
    final IndexTuningConfig tuningConfig = new IndexTuningConfig(null, null, null, 100, 2000L, null, null, null, null, null, new DynamicPartitionsSpec(1000, 2000L), new IndexSpec(new RoaringBitmapSerdeFactory(false), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), null, null, true, null, null, 100L, OffHeapMemorySegmentWriteOutMediumFactory.instance(), true, 10, 100, null, null);
}