Examples with ScanBasedDocIdIterator com.linkedin.pinot.core.operator.dociditerators.ScanBasedDocIdIterator used on opensource projects

Example 1 with ScanBasedDocIdIterator

use of com.linkedin.pinot.core.operator.dociditerators.ScanBasedDocIdIterator in project pinot by linkedin.

the class AndBlockDocIdSet method fastIterator.

public BlockDocIdIterator fastIterator() {
    long start = System.currentTimeMillis();
    List<List<IntPair>> sortedRangeSets = new ArrayList<>();
    List<ImmutableRoaringBitmap> childBitmaps = new ArrayList<ImmutableRoaringBitmap>();
    List<FilterBlockDocIdSet> scanBasedDocIdSets = new ArrayList<>();
    List<BlockDocIdIterator> remainingIterators = new ArrayList<>();
    for (BlockDocIdSet docIdSet : blockDocIdSets) {
        if (docIdSet instanceof SortedDocIdSet) {
            SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
            List<IntPair> pairs = sortedDocIdSet.getRaw();
            sortedRangeSets.add(pairs);
        } else if (docIdSet instanceof BitmapDocIdSet) {
            BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
            ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
            childBitmaps.add(childBitmap);
        } else if (docIdSet instanceof ScanBasedSingleValueDocIdSet) {
            scanBasedDocIdSets.add((ScanBasedSingleValueDocIdSet) docIdSet);
        } else if (docIdSet instanceof ScanBasedMultiValueDocIdSet) {
            scanBasedDocIdSets.add((ScanBasedMultiValueDocIdSet) docIdSet);
        } else {
            // TODO:handle child OR/AND as bitmap if possible
            remainingIterators.add(docIdSet.iterator());
        }
    }
    if (childBitmaps.size() == 0 && sortedRangeSets.size() == 0) {
        // When one or more of the operands are operators themselves, then we don't have a sorted or
        // bitmap index. In that case, just use the AndDocIdIterator to iterate over all of of the subtree.
        BlockDocIdIterator[] docIdIterators = new BlockDocIdIterator[blockDocIdSets.size()];
        for (int srcId = 0; srcId < blockDocIdSets.size(); srcId++) {
            docIdIterators[srcId] = blockDocIdSets.get(srcId).iterator();
        }
        return new AndDocIdIterator(docIdIterators);
    } else {
        // TODO: will be nice to re-order sorted and bitmap index based on size
        if (sortedRangeSets.size() > 0) {
            List<IntPair> pairList;
            pairList = SortedRangeIntersection.intersectSortedRangeSets(sortedRangeSets);
            answer = new MutableRoaringBitmap();
            for (IntPair pair : pairList) {
                // end is exclusive
                answer.add(pair.getLeft(), pair.getRight() + 1);
            }
        }
        // handle bitmaps
        if (childBitmaps.size() > 0) {
            if (answer == null) {
                answer = childBitmaps.get(0).toMutableRoaringBitmap();
                for (int i = 1; i < childBitmaps.size(); i++) {
                    answer.and(childBitmaps.get(i));
                }
            } else {
                for (int i = 0; i < childBitmaps.size(); i++) {
                    answer.and(childBitmaps.get(i));
                }
            }
        }
        // At this point, we must have 'answer' to be non-null.
        assert (answer != null) : "sortedRangeSets=" + sortedRangeSets.size() + ",childBitmaps=" + childBitmaps.size();
        // handle raw iterators
        for (FilterBlockDocIdSet scanBasedDocIdSet : scanBasedDocIdSets) {
            ScanBasedDocIdIterator iterator = (ScanBasedDocIdIterator) scanBasedDocIdSet.iterator();
            MutableRoaringBitmap scanAnswer = iterator.applyAnd(answer);
            answer.and(scanAnswer);
        }
        long end = System.currentTimeMillis();
        LOGGER.debug("Time to evaluate and Filter:{}", (end - start));
        // if other iterators exists resort to iterator style intersection
        BlockDocIdIterator answerDocIdIterator = new RangelessBitmapDocIdIterator(answer.getIntIterator());
        if (remainingIterators.size() == 0) {
            return answerDocIdIterator;
        } else {
            BlockDocIdIterator[] docIdIterators = new BlockDocIdIterator[remainingIterators.size() + 1];
            docIdIterators[0] = answerDocIdIterator;
            for (int i = 0; i < remainingIterators.size(); i++) {
                docIdIterators[i + 1] = remainingIterators.get(i);
            }
            return new AndDocIdIterator(docIdIterators);
        }
    }
}