Examples with RoaringBitmapSerdeFactory io.druid.segment.data.RoaringBitmapSerdeFactory used on opensource projects

Example 1 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class DictionaryEncodedColumnPartSerdeTest method testSerde.

@Test
public void testSerde() throws Exception {
    //bitmapSerdeFactory not specified
    String json = "{\n" + " \"type\": \"stringDictionary\",\n" + " \"byteOrder\": \"BIG_ENDIAN\"\n" + "}\n";
    ObjectMapper mapper = TestHelper.getObjectMapper();
    DictionaryEncodedColumnPartSerde serde = (DictionaryEncodedColumnPartSerde) mapper.readValue(mapper.writeValueAsString(mapper.readValue(json, ColumnPartSerde.class)), ColumnPartSerde.class);
    Assert.assertEquals(ByteOrder.BIG_ENDIAN, serde.getByteOrder());
    Assert.assertTrue(serde.getBitmapSerdeFactory() instanceof ConciseBitmapSerdeFactory);
    //bitmapSerdeFactory specified
    json = "{\n" + "\"type\": \"stringDictionary\",\n" + "\"byteOrder\": \"LITTLE_ENDIAN\",\n" + "\"bitmapSerdeFactory\": { \"type\": \"roaring\" }\n" + "}";
    serde = (DictionaryEncodedColumnPartSerde) mapper.readValue(mapper.writeValueAsString(mapper.readValue(json, ColumnPartSerde.class)), ColumnPartSerde.class);
    Assert.assertEquals(ByteOrder.LITTLE_ENDIAN, serde.getByteOrder());
    Assert.assertTrue(serde.getBitmapSerdeFactory() instanceof RoaringBitmapSerdeFactory);
}

Example 2 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class LikeFilterBenchmark method setup.

@Setup
public void setup() throws IOException {
    step = (END_INT - START_INT) / cardinality;
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new BitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add((i - START_INT) / step);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new BitmapIndexSelector() {

        @Override
        public Indexed<String> getDimensionValues(String dimension) {
            return dictionary;
        }

        @Override
        public int getNumRows() {
            throw new UnsupportedOperationException();
        }

        @Override
        public BitmapFactory getBitmapFactory() {
            return bitmapFactory;
        }

        @Override
        public ImmutableBitmap getBitmapIndex(String dimension, String value) {
            return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
        }

        @Override
        public BitmapIndex getBitmapIndex(String dimension) {
            return bitmapIndex;
        }

        @Override
        public ImmutableRTree getSpatialIndex(String dimension) {
            throw new UnsupportedOperationException();
        }
    };
}

Example 3 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class DimensionPredicateFilterBenchmark method setup.

@Setup
public void setup() throws IOException {
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new BitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add(i - START_INT);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new BitmapIndexSelector() {

        @Override
        public Indexed<String> getDimensionValues(String dimension) {
            return dictionary;
        }

        @Override
        public int getNumRows() {
            throw new UnsupportedOperationException();
        }

        @Override
        public BitmapFactory getBitmapFactory() {
            return bitmapFactory;
        }

        @Override
        public ImmutableBitmap getBitmapIndex(String dimension, String value) {
            return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
        }

        @Override
        public BitmapIndex getBitmapIndex(String dimension) {
            return bitmapIndex;
        }

        @Override
        public ImmutableRTree getSpatialIndex(String dimension) {
            throw new UnsupportedOperationException();
        }
    };
}

Example 4 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class BoundFilterBenchmark method setup.

@Setup
public void setup() throws IOException {
    step = (END_INT - START_INT) / cardinality;
    final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
    final BitmapSerdeFactory serdeFactory = new RoaringBitmapSerdeFactory(null);
    final List<Integer> ints = generateInts();
    final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, String>() {

        @Override
        public String apply(Integer i) {
            return i.toString();
        }
    }), GenericIndexed.STRING_STRATEGY);
    final BitmapIndex bitmapIndex = new BitmapIndexColumnPartSupplier(bitmapFactory, GenericIndexed.fromIterable(FluentIterable.from(ints).transform(new Function<Integer, ImmutableBitmap>() {

        @Override
        public ImmutableBitmap apply(Integer i) {
            final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
            mutableBitmap.add((i - START_INT) / step);
            return bitmapFactory.makeImmutableBitmap(mutableBitmap);
        }
    }), serdeFactory.getObjectStrategy()), dictionary).get();
    selector = new BitmapIndexSelector() {

        @Override
        public Indexed<String> getDimensionValues(String dimension) {
            return dictionary;
        }

        @Override
        public int getNumRows() {
            throw new UnsupportedOperationException();
        }

        @Override
        public BitmapFactory getBitmapFactory() {
            return bitmapFactory;
        }

        @Override
        public ImmutableBitmap getBitmapIndex(String dimension, String value) {
            return bitmapIndex.getBitmap(bitmapIndex.getIndex(value));
        }

        @Override
        public BitmapIndex getBitmapIndex(String dimension) {
            return bitmapIndex;
        }

        @Override
        public ImmutableRTree getSpatialIndex(String dimension) {
            throw new UnsupportedOperationException();
        }
    };
}

Example 5 with RoaringBitmapSerdeFactory

use of io.druid.segment.data.RoaringBitmapSerdeFactory in project druid by druid-io.

the class HadoopConverterJobTest method testSimpleJob.

@Test
public void testSimpleJob() throws IOException, InterruptedException {
    final SQLMetadataSegmentManager manager = new SQLMetadataSegmentManager(HadoopDruidConverterConfig.jsonMapper, new Supplier<MetadataSegmentManagerConfig>() {

        @Override
        public MetadataSegmentManagerConfig get() {
            return new MetadataSegmentManagerConfig();
        }
    }, metadataStorageTablesConfigSupplier, connector);
    final List<DataSegment> oldSemgments = getDataSegments(manager);
    final File tmpDir = temporaryFolder.newFolder();
    final HadoopConverterJob converterJob = new HadoopConverterJob(new HadoopDruidConverterConfig(DATASOURCE, interval, new IndexSpec(new RoaringBitmapSerdeFactory(null), CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, CompressionFactory.LongEncodingStrategy.LONGS), oldSemgments, true, tmpDir.toURI(), ImmutableMap.<String, String>of(), null, tmpSegmentDir.toURI().toString()));
    final List<DataSegment> segments = Lists.newArrayList(converterJob.run());
    Assert.assertNotNull("bad result", segments);
    Assert.assertEquals("wrong segment count", 4, segments.size());
    Assert.assertTrue(converterJob.getLoadedBytes() > 0);
    Assert.assertTrue(converterJob.getWrittenBytes() > 0);
    Assert.assertTrue(converterJob.getWrittenBytes() > converterJob.getLoadedBytes());
    Assert.assertEquals(oldSemgments.size(), segments.size());
    final DataSegment segment = segments.get(0);
    Assert.assertTrue(interval.contains(segment.getInterval()));
    Assert.assertTrue(segment.getVersion().endsWith("_converted"));
    Assert.assertTrue(segment.getLoadSpec().get("path").toString().contains("_converted"));
    for (File file : tmpDir.listFiles()) {
        Assert.assertFalse(file.isDirectory());
        Assert.assertTrue(file.isFile());
    }
    final Comparator<DataSegment> segmentComparator = new Comparator<DataSegment>() {

        @Override
        public int compare(DataSegment o1, DataSegment o2) {
            return o1.getIdentifier().compareTo(o2.getIdentifier());
        }
    };
    Collections.sort(oldSemgments, segmentComparator);
    Collections.sort(segments, segmentComparator);
    for (int i = 0; i < oldSemgments.size(); ++i) {
        final DataSegment oldSegment = oldSemgments.get(i);
        final DataSegment newSegment = segments.get(i);
        Assert.assertEquals(oldSegment.getDataSource(), newSegment.getDataSource());
        Assert.assertEquals(oldSegment.getInterval(), newSegment.getInterval());
        Assert.assertEquals(Sets.<String>newHashSet(oldSegment.getMetrics()), Sets.<String>newHashSet(newSegment.getMetrics()));
        Assert.assertEquals(Sets.<String>newHashSet(oldSegment.getDimensions()), Sets.<String>newHashSet(newSegment.getDimensions()));
        Assert.assertEquals(oldSegment.getVersion() + "_converted", newSegment.getVersion());
        Assert.assertTrue(oldSegment.getSize() < newSegment.getSize());
        Assert.assertEquals(oldSegment.getBinaryVersion(), newSegment.getBinaryVersion());
    }
}