Examples with IntIterator org.roaringbitmap.IntIterator used on opensource projects

Example 6 with IntIterator

use of org.roaringbitmap.IntIterator in project pinot by linkedin.

the class MVScanDocIdIterator method applyAnd.

@Override
public MutableRoaringBitmap applyAnd(MutableRoaringBitmap answer) {
    MutableRoaringBitmap result = new MutableRoaringBitmap();
    if (evaluator.alwaysFalse()) {
        return result;
    }
    IntIterator intIterator = answer.getIntIterator();
    int docId = -1, length;
    while (intIterator.hasNext() && docId < endDocId) {
        docId = intIterator.next();
        if (docId >= startDocId) {
            valueIterator.skipTo(docId);
            _numEntriesScanned++;
            length = valueIterator.nextIntVal(intArray);
            if (evaluator.apply(intArray, length)) {
                result.add(docId);
            }
        }
    }
    return result;
}

Example 7 with IntIterator

use of org.roaringbitmap.IntIterator in project pinot by linkedin.

the class AndBlockDocIdSet method slowIterator.

public BlockDocIdIterator slowIterator() {
    List<BlockDocIdIterator> rawIterators = new ArrayList<>();
    boolean useBitmapBasedIntersection = false;
    for (BlockDocIdSet docIdSet : blockDocIdSets) {
        if (docIdSet instanceof BitmapDocIdSet) {
            useBitmapBasedIntersection = true;
        }
    }
    BlockDocIdIterator[] docIdIterators;
    if (useBitmapBasedIntersection) {
        List<ImmutableRoaringBitmap> allBitmaps = new ArrayList<ImmutableRoaringBitmap>();
        for (BlockDocIdSet docIdSet : blockDocIdSets) {
            if (docIdSet instanceof SortedDocIdSet) {
                MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
                SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
                List<IntPair> pairs = sortedDocIdSet.getRaw();
                for (IntPair pair : pairs) {
                    // add takes [start, end) i.e inclusive
                    bitmap.add(pair.getLeft(), pair.getRight() + 1);
                // start, exclusive end.
                }
                allBitmaps.add(bitmap);
            } else if (docIdSet instanceof BitmapDocIdSet) {
                BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
                ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
                allBitmaps.add(childBitmap);
            } else {
                BlockDocIdIterator iterator = docIdSet.iterator();
                rawIterators.add(iterator);
            }
        }
        IntIterator intIterator;
        if (allBitmaps.size() > 1) {
            MutableRoaringBitmap answer = (MutableRoaringBitmap) allBitmaps.get(0).clone();
            for (int i = 1; i < allBitmaps.size(); i++) {
                answer.and(allBitmaps.get(i));
            }
            intIterator = answer.getIntIterator();
        } else {
            intIterator = allBitmaps.get(0).getIntIterator();
        }
        BitmapDocIdIterator singleBitmapBlockIdIterator = new BitmapDocIdIterator(intIterator);
        singleBitmapBlockIdIterator.setStartDocId(minDocId);
        singleBitmapBlockIdIterator.setEndDocId(maxDocId);
        rawIterators.add(0, singleBitmapBlockIdIterator);
        docIdIterators = new BlockDocIdIterator[rawIterators.size()];
        rawIterators.toArray(docIdIterators);
    } else {
        docIdIterators = new BlockDocIdIterator[blockDocIdSets.size()];
        for (int srcId = 0; srcId < blockDocIdSets.size(); srcId++) {
            docIdIterators[srcId] = blockDocIdSets.get(srcId).iterator();
        }
    }
    return new AndDocIdIterator(docIdIterators);
}

Example 8 with IntIterator

use of org.roaringbitmap.IntIterator in project pinot by linkedin.

the class RealtimeSegmentImpl method getSortedBitmapIntIteratorsForLongColumn.

private IntIterator[] getSortedBitmapIntIteratorsForLongColumn(final String columnToSortOn) {
    final RealtimeInvertedIndex index = invertedIndexMap.get(columnToSortOn);
    final MutableDictionaryReader dictionary = dictionaryMap.get(columnToSortOn);
    final IntIterator[] intIterators = new IntIterator[dictionary.length()];
    final List<Long> rawValues = new ArrayList<Long>();
    for (int i = 0; i < dictionary.length(); i++) {
        rawValues.add((Long) dictionary.get(i));
    }
    long start = System.currentTimeMillis();
    Collections.sort(rawValues);
    LOGGER.info("Column {}, dictionary len : {}, time to sort : {} ", columnToSortOn, dictionary.length(), (System.currentTimeMillis() - start));
    for (int i = 0; i < rawValues.size(); i++) {
        intIterators[i] = index.getDocIdSetFor(dictionary.indexOf(rawValues.get(i))).getIntIterator();
    }
    return intIterators;
}

Example 9 with IntIterator

use of org.roaringbitmap.IntIterator in project pinot by linkedin.

the class RealtimeSegmentImpl method getSortedDocIdIterationOrderWithSortedColumn.

/**
   * Returns the docIds to use for iteration when the data is sorted by <code>columnToSortOn</code>
   * @param columnToSortOn The column to use for sorting
   * @return The docIds to use for iteration
   */
public int[] getSortedDocIdIterationOrderWithSortedColumn(final String columnToSortOn) {
    int[] docIds = new int[numDocsIndexed];
    final IntIterator[] iterators;
    // Get docId iterators that iterate in order on the data
    switch(dataSchema.getFieldSpecFor(columnToSortOn).getDataType()) {
        case INT:
            iterators = getSortedBitmapIntIteratorsForIntegerColumn(columnToSortOn);
            break;
        case LONG:
            iterators = getSortedBitmapIntIteratorsForLongColumn(columnToSortOn);
            break;
        case FLOAT:
            iterators = getSortedBitmapIntIteratorsForFloatColumn(columnToSortOn);
            break;
        case DOUBLE:
            iterators = getSortedBitmapIntIteratorsForDoubleColumn(columnToSortOn);
            break;
        case STRING:
        case BOOLEAN:
            iterators = getSortedBitmapIntIteratorsForStringColumn(columnToSortOn);
            break;
        default:
            iterators = null;
            break;
    }
    // Drain the iterators into the docIds array
    int i = 0;
    for (IntIterator iterator : iterators) {
        while (iterator.hasNext()) {
            docIds[i] = iterator.next();
            ++i;
        }
    }
    // Sanity check
    if (i != numDocsIndexed) {
        throw new RuntimeException("The number of docs indexed is not equal to the number of sorted documents");
    }
    return docIds;
}

Example 10 with IntIterator

use of org.roaringbitmap.IntIterator in project druid by druid-io.

the class ImmutableRTreeTest method testSearchWithSplitLimitedBound.

@Test
public void testSearchWithSplitLimitedBound() {
    BitmapFactory bf = new ConciseBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    tree.insert(new float[] { 0, 0 }, 1);
    tree.insert(new float[] { 1, 3 }, 2);
    tree.insert(new float[] { 4, 2 }, 3);
    tree.insert(new float[] { 5, 0 }, 4);
    tree.insert(new float[] { -4, -3 }, 5);
    Random rand = new Random();
    for (int i = 0; i < 4995; i++) {
        tree.insert(new float[] { (float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0) }, i);
    }
    ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree);
    Iterable<ImmutableBitmap> points = searchTree.search(new RadiusBound(new float[] { 0, 0 }, 5, 2));
    ImmutableBitmap finalSet = bf.union(points);
    Assert.assertTrue(finalSet.size() >= 5);
    Set<Integer> expected = Sets.newHashSet(1, 2, 3, 4, 5);
    IntIterator iter = finalSet.iterator();
    while (iter.hasNext()) {
        Assert.assertTrue(expected.contains(iter.next()));
    }
}