Examples with BitmapFactory org.apache.druid.collections.bitmap.BitmapFactory used on opensource projects

Example 31 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ImmutableRTreeTest method showBenchmarksBoundWithLimits.

// TODO rewrite to JMH and move to the benchmarks project
@SuppressWarnings("unused")
public void showBenchmarksBoundWithLimits() {
    // final int start = 1;
    final int start = 10000000;
    final int factor = 10;
    final int end = 10000000;
    for (int numPoints = start; numPoints <= end; numPoints *= factor) {
        try {
            BitmapFactory bf = new ConciseBitmapFactory();
            RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
            Stopwatch stopwatch = Stopwatch.createStarted();
            Random rand = ThreadLocalRandom.current();
            for (int i = 0; i < numPoints; i++) {
                tree.insert(new float[] { (float) (rand.nextDouble() * 100), (float) (rand.nextDouble() * 100) }, i);
            }
            long stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            System.out.printf(Locale.ENGLISH, "[%,d]: insert = %,d ms%n", numPoints, stop);
            stopwatch.reset().start();
            ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
            stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            System.out.printf(Locale.ENGLISH, "[%,d]: size = %,d bytes%n", numPoints, searchTree.toBytes().length);
            System.out.printf(Locale.ENGLISH, "[%,d]: buildImmutable = %,d ms%n", numPoints, stop);
            stopwatch.reset().start();
            Iterable<ImmutableBitmap> points = searchTree.search(new RectangularBound(new float[] { 40, 40 }, new float[] { 60, 60 }, 100));
            Iterables.size(points);
            stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            System.out.printf(Locale.ENGLISH, "[%,d]: search = %,dms%n", numPoints, stop);
            stopwatch.reset().start();
            ImmutableBitmap finalSet = bf.union(points);
            stop = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            System.out.printf(Locale.ENGLISH, "[%,d]: union of %,d points in %,d ms%n", numPoints, finalSet.size(), stop);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}

Example 32 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ImmutableRTreeTest method testSearchWithSplit4Roaring.

@Test
public void testSearchWithSplit4Roaring() {
    BitmapFactory bf = new RoaringBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    Random rand = ThreadLocalRandom.current();
    int outPolygon = 0, inPolygon = 0;
    for (; inPolygon < 500; ) {
        double abscissa = rand.nextDouble() * 5;
        double ordinate = rand.nextDouble() * 4;
        if (abscissa < 1 || abscissa > 4 || ordinate < 1 || ordinate > 3 || abscissa < 2 && ordinate > 2) {
            tree.insert(new float[] { (float) abscissa, (float) ordinate }, outPolygon + 500);
            outPolygon++;
        } else if (abscissa > 1 && abscissa < 4 && ordinate > 1 && ordinate < 2 || abscissa > 2 && abscissa < 4 && ordinate >= 2 && ordinate < 3) {
            tree.insert(new float[] { (float) abscissa, (float) ordinate }, inPolygon);
            inPolygon++;
        }
    }
    ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
    Iterable<ImmutableBitmap> points = searchTree.search(PolygonBound.from(new float[] { 1.0f, 1.0f, 2.0f, 2.0f, 4.0f, 4.0f }, new float[] { 1.0f, 2.0f, 2.0f, 3.0f, 3.0f, 1.0f }));
    ImmutableBitmap finalSet = bf.union(points);
    Assert.assertTrue(finalSet.size() == 500);
    Set<Integer> expected = new HashSet<>();
    for (int i = 0; i < 500; i++) {
        expected.add(i);
    }
    IntIterator iter = finalSet.iterator();
    while (iter.hasNext()) {
        Assert.assertTrue(expected.contains(iter.next()));
    }
}

Example 33 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class LinearGutmanSplitStrategyTest method testPickSeedsRoaring.

@Test
public void testPickSeedsRoaring() {
    BitmapFactory bf = new RoaringBitmapFactory();
    LinearGutmanSplitStrategy strategy = new LinearGutmanSplitStrategy(0, 50, bf);
    Node node = new Node(new float[2], new float[2], true, bf);
    node.addChild(new Point(new float[] { 3, 7 }, 1, bf));
    node.addChild(new Point(new float[] { 1, 6 }, 1, bf));
    node.addChild(new Point(new float[] { 9, 8 }, 1, bf));
    node.addChild(new Point(new float[] { 2, 5 }, 1, bf));
    node.addChild(new Point(new float[] { 4, 4 }, 1, bf));
    node.enclose();
    Node[] groups = strategy.split(node);
    Assert.assertEquals(groups[0].getMinCoordinates()[0], 1.0f);
    Assert.assertEquals(groups[0].getMinCoordinates()[1], 4.0f);
    Assert.assertEquals(groups[1].getMinCoordinates()[0], 9.0f);
    Assert.assertEquals(groups[1].getMinCoordinates()[1], 8.0f);
}

Example 34 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class DictionaryEncodedColumnMerger method writeIndexes.

@Override
public void writeIndexes(@Nullable List<IntBuffer> segmentRowNumConversions) throws IOException {
    if (!capabilities.hasBitmapIndexes()) {
        return;
    }
    long dimStartTime = System.currentTimeMillis();
    final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
    String bmpFilename = StringUtils.format("%s.inverted", dimensionName);
    bitmapWriter = new GenericIndexedWriter<>(segmentWriteOutMedium, bmpFilename, indexSpec.getBitmapSerdeFactory().getObjectStrategy());
    bitmapWriter.open();
    bitmapWriter.setObjectsNotSorted();
    BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory();
    ExtendedIndexesMerger extendedIndexesMerger = getExtendedIndexesMerger();
    if (extendedIndexesMerger != null) {
        extendedIndexesMerger.initialize();
    }
    IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName);
    // Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result.
    for (int dictId = 0; dictId < dictionarySize; dictId++) {
        progress.progress();
        MutableBitmap mergedIndexes = mergeBitmaps(segmentRowNumConversions, bitmapFactory, dictIdSeeker, dictId);
        if (extendedIndexesMerger != null) {
            extendedIndexesMerger.mergeIndexes(dictId, mergedIndexes);
        }
    }
    if (extendedIndexesMerger != null) {
        extendedIndexesMerger.write();
    }
    log.debug("Completed dim[%s] inverted with cardinality[%,d] in %,d millis.", dimensionName, dictionarySize, System.currentTimeMillis() - dimStartTime);
    if (dictionaryMergeIterator != null) {
        dictionaryMergeIterator.close();
    }
}

Example 35 with BitmapFactory

use of org.apache.druid.collections.bitmap.BitmapFactory in project druid by druid-io.

the class ColumnSelectorBitmapIndexSelectorTest method setup.

@Before
public void setup() {
    bitmapFactory = EasyMock.createMock(BitmapFactory.class);
    virtualColumns = EasyMock.createMock(VirtualColumns.class);
    index = EasyMock.createMock(ColumnSelector.class);
    bitmapIndexSelector = new ColumnSelectorBitmapIndexSelector(bitmapFactory, virtualColumns, index);
    EasyMock.expect(virtualColumns.getVirtualColumn(STRING_DICTIONARY_COLUMN_NAME)).andReturn(null).anyTimes();
    EasyMock.expect(virtualColumns.getVirtualColumn(NON_STRING_DICTIONARY_COLUMN_NAME)).andReturn(null).anyTimes();
    ColumnHolder holder = EasyMock.createMock(ColumnHolder.class);
    EasyMock.expect(index.getColumnHolder(STRING_DICTIONARY_COLUMN_NAME)).andReturn(holder).anyTimes();
    StringDictionaryEncodedColumn stringColumn = EasyMock.createMock(StringDictionaryEncodedColumn.class);
    EasyMock.expect(holder.getCapabilities()).andReturn(ColumnCapabilitiesImpl.createDefault().setType(ColumnType.STRING).setDictionaryEncoded(true).setDictionaryValuesUnique(true).setDictionaryValuesSorted(true).setHasBitmapIndexes(true)).anyTimes();
    EasyMock.expect(holder.getColumn()).andReturn(stringColumn).anyTimes();
    BitmapIndex someIndex = EasyMock.createMock(BitmapIndex.class);
    EasyMock.expect(holder.getBitmapIndex()).andReturn(someIndex).anyTimes();
    ImmutableBitmap someBitmap = EasyMock.createMock(ImmutableBitmap.class);
    EasyMock.expect(someIndex.getIndex("foo")).andReturn(0).anyTimes();
    EasyMock.expect(someIndex.getBitmap(0)).andReturn(someBitmap).anyTimes();
    ColumnHolder nonStringHolder = EasyMock.createMock(ColumnHolder.class);
    EasyMock.expect(index.getColumnHolder(NON_STRING_DICTIONARY_COLUMN_NAME)).andReturn(nonStringHolder).anyTimes();
    EasyMock.expect(nonStringHolder.getCapabilities()).andReturn(ColumnCapabilitiesImpl.createDefault().setType(ColumnType.ofComplex("testBlob")).setDictionaryEncoded(true).setDictionaryValuesUnique(true).setDictionaryValuesSorted(true).setHasBitmapIndexes(true).setFilterable(true)).anyTimes();
    EasyMock.replay(bitmapFactory, virtualColumns, index, holder, stringColumn, nonStringHolder, someIndex, someBitmap);
}