Examples with RoaringBitmapSerdeFactory io.druid.segment.data.RoaringBitmapSerdeFactory used on opensource projects

Example 6 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class HadoopConverterJobTest method testHadoopFailure.

@Test
// This takes a long time due to retries
@Ignore
public void testHadoopFailure() throws IOException, InterruptedException {
    final SQLMetadataSegmentManager manager = new SQLMetadataSegmentManager(HadoopDruidConverterConfig.jsonMapper, new Supplier<MetadataSegmentManagerConfig>() {

        @Override
        public MetadataSegmentManagerConfig get() {
            return new MetadataSegmentManagerConfig();
        }
    }, metadataStorageTablesConfigSupplier, connector);
    final List<DataSegment> oldSemgments = getDataSegments(manager);
    final File tmpDir = temporaryFolder.newFolder();
    final HadoopConverterJob converterJob = new HadoopConverterJob(new HadoopDruidConverterConfig(DATASOURCE, interval, new IndexSpec(new RoaringBitmapSerdeFactory(null), CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressionFactory.LongEncodingStrategy.LONGS), oldSemgments, true, tmpDir.toURI(), ImmutableMap.<String, String>of(), null, tmpSegmentDir.toURI().toString()));
    corrupt(oldSemgments.get(0));
    final List<DataSegment> result = converterJob.run();
    Assert.assertNull("result should be null", result);
    final List<DataSegment> segments = getDataSegments(manager);
    Assert.assertEquals(oldSemgments.size(), segments.size());
    Assert.assertEquals(oldSemgments, segments);
}

Example 7 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class BaseFilterTest method makeConstructors.

public static Collection<Object[]> makeConstructors() {
    final List<Object[]> constructors = Lists.newArrayList();
    final Map<String, BitmapSerdeFactory> bitmapSerdeFactories = ImmutableMap.<String, BitmapSerdeFactory>of("concise", new ConciseBitmapSerdeFactory(), "roaring", new RoaringBitmapSerdeFactory(true));
    final Map<String, IndexMerger> indexMergers = ImmutableMap.<String, IndexMerger>of("IndexMerger", TestHelper.getTestIndexMerger(), "IndexMergerV9", TestHelper.getTestIndexMergerV9());
    final Map<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finishers = ImmutableMap.of("incremental", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {

        @Override
        public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
            final IncrementalIndex index = input.buildIncrementalIndex();
            return Pair.<StorageAdapter, Closeable>of(new IncrementalIndexStorageAdapter(index), new Closeable() {

                @Override
                public void close() throws IOException {
                    index.close();
                }
            });
        }
    }, "mmapped", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {

        @Override
        public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
            final QueryableIndex index = input.buildMMappedIndex();
            return Pair.<StorageAdapter, Closeable>of(new QueryableIndexStorageAdapter(index), new Closeable() {

                @Override
                public void close() throws IOException {
                    index.close();
                }
            });
        }
    }, "mmappedMerged", new Function<IndexBuilder, Pair<StorageAdapter, Closeable>>() {

        @Override
        public Pair<StorageAdapter, Closeable> apply(IndexBuilder input) {
            final QueryableIndex index = input.buildMMappedMergedIndex();
            return Pair.<StorageAdapter, Closeable>of(new QueryableIndexStorageAdapter(index), new Closeable() {

                @Override
                public void close() throws IOException {
                    index.close();
                }
            });
        }
    });
    for (Map.Entry<String, BitmapSerdeFactory> bitmapSerdeFactoryEntry : bitmapSerdeFactories.entrySet()) {
        for (Map.Entry<String, IndexMerger> indexMergerEntry : indexMergers.entrySet()) {
            for (Map.Entry<String, Function<IndexBuilder, Pair<StorageAdapter, Closeable>>> finisherEntry : finishers.entrySet()) {
                for (boolean cnf : ImmutableList.of(false, true)) {
                    for (boolean optimize : ImmutableList.of(false, true)) {
                        final String testName = String.format("bitmaps[%s], indexMerger[%s], finisher[%s], optimize[%s]", bitmapSerdeFactoryEntry.getKey(), indexMergerEntry.getKey(), finisherEntry.getKey(), optimize);
                        final IndexBuilder indexBuilder = IndexBuilder.create().indexSpec(new IndexSpec(bitmapSerdeFactoryEntry.getValue(), null, null, null)).indexMerger(indexMergerEntry.getValue());
                        constructors.add(new Object[] { testName, indexBuilder, finisherEntry.getValue(), cnf, optimize });
                    }
                }
            }
        }
    }
    return constructors;
}

Example 8 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class IndexSpecTest method testSerde.

@Test
public void testSerde() throws Exception {
    final ObjectMapper objectMapper = new DefaultObjectMapper();
    final String json = "{ \"bitmap\" : { \"type\" : \"roaring\" }, \"dimensionCompression\" : \"lz4\", \"metricCompression\" : \"lzf\"" + ", \"longEncoding\" : \"auto\" }";
    final IndexSpec spec = objectMapper.readValue(json, IndexSpec.class);
    Assert.assertEquals(new RoaringBitmapSerdeFactory(null), spec.getBitmapSerdeFactory());
    Assert.assertEquals(CompressedObjectStrategy.CompressionStrategy.LZ4, spec.getDimensionCompression());
    Assert.assertEquals(CompressedObjectStrategy.CompressionStrategy.LZF, spec.getMetricCompression());
    Assert.assertEquals(CompressionFactory.LongEncodingStrategy.AUTO, spec.getLongEncoding());
    Assert.assertEquals(spec, objectMapper.readValue(objectMapper.writeValueAsBytes(spec), IndexSpec.class));
}

Example 9 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class TaskSerdeTest method testSegmentConvertSerde.

@Test
public void testSegmentConvertSerde() throws IOException {
    final DataSegment segment = new DataSegment("dataSource", Interval.parse("1990-01-01/1999-12-31"), "version", ImmutableMap.<String, Object>of(), ImmutableList.of("dim1", "dim2"), ImmutableList.of("metric1", "metric2"), NoneShardSpec.instance(), 0, 12345L);
    final ConvertSegmentTask convertSegmentTaskOriginal = ConvertSegmentTask.create(segment, new IndexSpec(new RoaringBitmapSerdeFactory(null), CompressedObjectStrategy.CompressionStrategy.LZF, CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressionFactory.LongEncodingStrategy.LONGS), false, true, null);
    final String json = jsonMapper.writeValueAsString(convertSegmentTaskOriginal);
    final Task task = jsonMapper.readValue(json, Task.class);
    Assert.assertTrue(task instanceof ConvertSegmentTask);
    final ConvertSegmentTask convertSegmentTask = (ConvertSegmentTask) task;
    Assert.assertEquals(convertSegmentTaskOriginal.getDataSource(), convertSegmentTask.getDataSource());
    Assert.assertEquals(convertSegmentTaskOriginal.getInterval(), convertSegmentTask.getInterval());
    Assert.assertEquals(convertSegmentTaskOriginal.getIndexSpec().getBitmapSerdeFactory().getClass().getCanonicalName(), convertSegmentTask.getIndexSpec().getBitmapSerdeFactory().getClass().getCanonicalName());
    Assert.assertEquals(convertSegmentTaskOriginal.getIndexSpec().getDimensionCompression(), convertSegmentTask.getIndexSpec().getDimensionCompression());
    Assert.assertEquals(convertSegmentTaskOriginal.getIndexSpec().getMetricCompression(), convertSegmentTask.getIndexSpec().getMetricCompression());
    Assert.assertEquals(false, convertSegmentTask.isForce());
    Assert.assertEquals(segment, convertSegmentTask.getSegment());
}

Example 10 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class DumpSegment method runBitmaps.

private void runBitmaps(final Injector injector, final QueryableIndex index) throws IOException {
    final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
    final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
    final BitmapSerdeFactory bitmapSerdeFactory;
    if (bitmapFactory instanceof ConciseBitmapFactory) {
        bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
    } else if (bitmapFactory instanceof RoaringBitmapFactory) {
        bitmapSerdeFactory = new RoaringBitmapSerdeFactory(null);
    } else {
        throw new ISE("Don't know which BitmapSerdeFactory to use for BitmapFactory[%s]!", bitmapFactory.getClass().getName());
    }
    final List<String> columnNames = getColumnsToInclude(index);
    withOutputStream(new Function<OutputStream, Object>() {

        @Override
        public Object apply(final OutputStream out) {
            try {
                final JsonGenerator jg = objectMapper.getFactory().createGenerator(out);
                jg.writeStartObject();
                jg.writeObjectField("bitmapSerdeFactory", bitmapSerdeFactory);
                jg.writeFieldName("bitmaps");
                jg.writeStartObject();
                for (final String columnName : columnNames) {
                    final Column column = index.getColumn(columnName);
                    final BitmapIndex bitmapIndex = column.getBitmapIndex();
                    if (bitmapIndex == null) {
                        jg.writeNullField(columnName);
                    } else {
                        jg.writeFieldName(columnName);
                        jg.writeStartObject();
                        for (int i = 0; i < bitmapIndex.getCardinality(); i++) {
                            jg.writeFieldName(Strings.nullToEmpty(bitmapIndex.getValue(i)));
                            final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
                            if (decompressBitmaps) {
                                jg.writeStartArray();
                                final IntIterator iterator = bitmap.iterator();
                                while (iterator.hasNext()) {
                                    final int rowNum = iterator.next();
                                    jg.writeNumber(rowNum);
                                }
                                jg.writeEndArray();
                            } else {
                                jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap));
                            }
                        }
                        jg.writeEndObject();
                    }
                }
                jg.writeEndObject();
                jg.writeEndObject();
                jg.close();
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
            return null;
        }
    });
}