Examples with IntPair com.linkedin.pinot.common.utils.Pairs.IntPair used on opensource projects

Example 1 with IntPair

use of com.linkedin.pinot.common.utils.Pairs.IntPair in project pinot by linkedin.

the class OrDocIdIterator method advance.

@Override
public int advance(int targetDocId) {
    if (currentDocId == Constants.EOF) {
        return Constants.EOF;
    }
    if (targetDocId < minDocId) {
        targetDocId = minDocId;
    } else if (targetDocId > maxDocId) {
        currentDocId = Constants.EOF;
        return currentDocId;
    }
    long start = System.nanoTime();
    // Remove iterators that are before the target document id from the queue
    Iterator<IntPair> iterator = queue.iterator();
    while (iterator.hasNext()) {
        IntPair pair = iterator.next();
        if (pair.getLeft() < targetDocId) {
            iterator.remove();
            iteratorIsInQueue[pair.getRight()] = false;
        }
    }
    // Advance all iterators that are not in the queue to the target document id
    for (int i = 0; i < docIdIterators.length; i++) {
        if (!iteratorIsInQueue[i]) {
            int nextDocId = docIdIterators[i].advance(targetDocId);
            if (nextDocId != Constants.EOF) {
                pointers[i].setLeft(nextDocId);
                queue.add(pointers[i]);
            }
            iteratorIsInQueue[i] = true;
        }
    }
    // Return the first element
    if (queue.size() > 0) {
        currentDocId = queue.peek().getLeft();
    } else {
        currentDocId = Constants.EOF;
    }
    long end = System.nanoTime();
    timeMeasure.addAndGet(end - start);
    return currentDocId;
}

Example 2 with IntPair

use of com.linkedin.pinot.common.utils.Pairs.IntPair in project pinot by linkedin.

the class AndBlockDocIdSet method slowIterator.

public BlockDocIdIterator slowIterator() {
    List<BlockDocIdIterator> rawIterators = new ArrayList<>();
    boolean useBitmapBasedIntersection = false;
    for (BlockDocIdSet docIdSet : blockDocIdSets) {
        if (docIdSet instanceof BitmapDocIdSet) {
            useBitmapBasedIntersection = true;
        }
    }
    BlockDocIdIterator[] docIdIterators;
    if (useBitmapBasedIntersection) {
        List<ImmutableRoaringBitmap> allBitmaps = new ArrayList<ImmutableRoaringBitmap>();
        for (BlockDocIdSet docIdSet : blockDocIdSets) {
            if (docIdSet instanceof SortedDocIdSet) {
                MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
                SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
                List<IntPair> pairs = sortedDocIdSet.getRaw();
                for (IntPair pair : pairs) {
                    // add takes [start, end) i.e inclusive
                    bitmap.add(pair.getLeft(), pair.getRight() + 1);
                // start, exclusive end.
                }
                allBitmaps.add(bitmap);
            } else if (docIdSet instanceof BitmapDocIdSet) {
                BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
                ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
                allBitmaps.add(childBitmap);
            } else {
                BlockDocIdIterator iterator = docIdSet.iterator();
                rawIterators.add(iterator);
            }
        }
        IntIterator intIterator;
        if (allBitmaps.size() > 1) {
            MutableRoaringBitmap answer = (MutableRoaringBitmap) allBitmaps.get(0).clone();
            for (int i = 1; i < allBitmaps.size(); i++) {
                answer.and(allBitmaps.get(i));
            }
            intIterator = answer.getIntIterator();
        } else {
            intIterator = allBitmaps.get(0).getIntIterator();
        }
        BitmapDocIdIterator singleBitmapBlockIdIterator = new BitmapDocIdIterator(intIterator);
        singleBitmapBlockIdIterator.setStartDocId(minDocId);
        singleBitmapBlockIdIterator.setEndDocId(maxDocId);
        rawIterators.add(0, singleBitmapBlockIdIterator);
        docIdIterators = new BlockDocIdIterator[rawIterators.size()];
        rawIterators.toArray(docIdIterators);
    } else {
        docIdIterators = new BlockDocIdIterator[blockDocIdSets.size()];
        for (int srcId = 0; srcId < blockDocIdSets.size(); srcId++) {
            docIdIterators[srcId] = blockDocIdSets.get(srcId).iterator();
        }
    }
    return new AndDocIdIterator(docIdIterators);
}

Example 3 with IntPair

use of com.linkedin.pinot.common.utils.Pairs.IntPair in project pinot by linkedin.

the class AndBlockDocIdSet method fastIterator.

public BlockDocIdIterator fastIterator() {
    long start = System.currentTimeMillis();
    List<List<IntPair>> sortedRangeSets = new ArrayList<>();
    List<ImmutableRoaringBitmap> childBitmaps = new ArrayList<ImmutableRoaringBitmap>();
    List<FilterBlockDocIdSet> scanBasedDocIdSets = new ArrayList<>();
    List<BlockDocIdIterator> remainingIterators = new ArrayList<>();
    for (BlockDocIdSet docIdSet : blockDocIdSets) {
        if (docIdSet instanceof SortedDocIdSet) {
            SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
            List<IntPair> pairs = sortedDocIdSet.getRaw();
            sortedRangeSets.add(pairs);
        } else if (docIdSet instanceof BitmapDocIdSet) {
            BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
            ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
            childBitmaps.add(childBitmap);
        } else if (docIdSet instanceof ScanBasedSingleValueDocIdSet) {
            scanBasedDocIdSets.add((ScanBasedSingleValueDocIdSet) docIdSet);
        } else if (docIdSet instanceof ScanBasedMultiValueDocIdSet) {
            scanBasedDocIdSets.add((ScanBasedMultiValueDocIdSet) docIdSet);
        } else {
            // TODO:handle child OR/AND as bitmap if possible
            remainingIterators.add(docIdSet.iterator());
        }
    }
    if (childBitmaps.size() == 0 && sortedRangeSets.size() == 0) {
        // When one or more of the operands are operators themselves, then we don't have a sorted or
        // bitmap index. In that case, just use the AndDocIdIterator to iterate over all of of the subtree.
        BlockDocIdIterator[] docIdIterators = new BlockDocIdIterator[blockDocIdSets.size()];
        for (int srcId = 0; srcId < blockDocIdSets.size(); srcId++) {
            docIdIterators[srcId] = blockDocIdSets.get(srcId).iterator();
        }
        return new AndDocIdIterator(docIdIterators);
    } else {
        // TODO: will be nice to re-order sorted and bitmap index based on size
        if (sortedRangeSets.size() > 0) {
            List<IntPair> pairList;
            pairList = SortedRangeIntersection.intersectSortedRangeSets(sortedRangeSets);
            answer = new MutableRoaringBitmap();
            for (IntPair pair : pairList) {
                // end is exclusive
                answer.add(pair.getLeft(), pair.getRight() + 1);
            }
        }
        // handle bitmaps
        if (childBitmaps.size() > 0) {
            if (answer == null) {
                answer = childBitmaps.get(0).toMutableRoaringBitmap();
                for (int i = 1; i < childBitmaps.size(); i++) {
                    answer.and(childBitmaps.get(i));
                }
            } else {
                for (int i = 0; i < childBitmaps.size(); i++) {
                    answer.and(childBitmaps.get(i));
                }
            }
        }
        // At this point, we must have 'answer' to be non-null.
        assert (answer != null) : "sortedRangeSets=" + sortedRangeSets.size() + ",childBitmaps=" + childBitmaps.size();
        // handle raw iterators
        for (FilterBlockDocIdSet scanBasedDocIdSet : scanBasedDocIdSets) {
            ScanBasedDocIdIterator iterator = (ScanBasedDocIdIterator) scanBasedDocIdSet.iterator();
            MutableRoaringBitmap scanAnswer = iterator.applyAnd(answer);
            answer.and(scanAnswer);
        }
        long end = System.currentTimeMillis();
        LOGGER.debug("Time to evaluate and Filter:{}", (end - start));
        // if other iterators exists resort to iterator style intersection
        BlockDocIdIterator answerDocIdIterator = new RangelessBitmapDocIdIterator(answer.getIntIterator());
        if (remainingIterators.size() == 0) {
            return answerDocIdIterator;
        } else {
            BlockDocIdIterator[] docIdIterators = new BlockDocIdIterator[remainingIterators.size() + 1];
            docIdIterators[0] = answerDocIdIterator;
            for (int i = 0; i < remainingIterators.size(); i++) {
                docIdIterators[i + 1] = remainingIterators.get(i);
            }
            return new AndDocIdIterator(docIdIterators);
        }
    }
}

Example 4 with IntPair

use of com.linkedin.pinot.common.utils.Pairs.IntPair in project pinot by linkedin.

the class StarTreeDataSorter method groupByIntColumnCount.

/**
   * Perform group-by based on the 'count' for the given column.
   * @param startDocId inclusive
   * @param endDocId exclusive
   * @param colIndex
   * @return start, end for each value. inclusive start, exclusive end
   */
public Map<Integer, IntPair> groupByIntColumnCount(int startDocId, int endDocId, Integer colIndex) {
    int length = endDocId - startDocId;
    Map<Integer, IntPair> rangeMap = new LinkedHashMap<>();
    final int startOffset = startDocId * totalSizeInBytes;
    int prevValue = -1;
    int prevStart = 0;
    byte[] dimBuff = new byte[dimensionSizeInBytes];
    for (int i = 0; i < length; i++) {
        mappedByteBuffer.copyTo(startOffset + (i * totalSizeInBytes), dimBuff, 0, dimensionSizeInBytes);
        int value = ByteBuffer.wrap(dimBuff).asIntBuffer().get(colIndex);
        if (prevValue != -1 && prevValue != value) {
            rangeMap.put(prevValue, new IntPair(startDocId + prevStart, startDocId + i));
            prevStart = i;
        }
        prevValue = value;
    }
    rangeMap.put(prevValue, new IntPair(startDocId + prevStart, endDocId));
    return rangeMap;
}

Example 5 with IntPair

use of com.linkedin.pinot.common.utils.Pairs.IntPair in project pinot by linkedin.

the class StarTreeDataTable method groupByIntColumnCount.

/**
   *
   * @param startDocId inclusive
   * @param endDocId exclusive
   * @param colIndex
   * @return start,end for each value. inclusive start, exclusive end
   */
public Int2ObjectMap<IntPair> groupByIntColumnCount(int startDocId, int endDocId, Integer colIndex) {
    MMapBuffer mappedByteBuffer = null;
    try {
        int length = endDocId - startDocId;
        Int2ObjectMap<IntPair> rangeMap = new Int2ObjectLinkedOpenHashMap<>();
        final long startOffset = startDocId * (long) totalSizeInBytes;
        mappedByteBuffer = new MMapBuffer(file, startOffset, length * (long) totalSizeInBytes, MMapMode.READ_WRITE);
        int prevValue = -1;
        int prevStart = 0;
        for (int i = 0; i < length; i++) {
            int value = flipEndiannessIfNeeded(mappedByteBuffer.getInt((i * (long) totalSizeInBytes) + (colIndex * V1Constants.Numbers.INTEGER_SIZE)));
            if (prevValue != -1 && prevValue != value) {
                rangeMap.put(prevValue, new IntPair(startDocId + prevStart, startDocId + i));
                prevStart = i;
            }
            prevValue = value;
        }
        rangeMap.put(prevValue, new IntPair(startDocId + prevStart, endDocId));
        return rangeMap;
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        if (mappedByteBuffer != null) {
            try {
                mappedByteBuffer.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    return EMPTY_INT_OBJECT_MAP;
}