Search in sources :

Example 26 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class AndBlockDocIdSet method iterator.

@Override
public BlockDocIdIterator iterator() {
    //TODO: Remove this validation code once we have enough testing
    if (validate) {
        BlockDocIdIterator slowIterator = slowIterator();
        BlockDocIdIterator fastIterator = fastIterator();
        List<Integer> matchedIds = new ArrayList<>();
        while (true) {
            int docId1 = slowIterator.next();
            int docId2 = fastIterator.next();
            if (docId1 != docId2) {
                LOGGER.error("ERROR docId1:" + docId1 + " docId2:" + docId2);
            } else {
                matchedIds.add(docId1);
            }
            if (docId1 == Constants.EOF || docId2 == Constants.EOF) {
                break;
            }
        }
        answer = null;
    }
    return fastIterator();
}
Also used : ArrayList(java.util.ArrayList) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator)

Example 27 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class OrBlockDocIdSet method iterator.

@Override
public BlockDocIdIterator iterator() {
    List<BlockDocIdIterator> rawIterators = new ArrayList<>();
    boolean useBitmapOr = false;
    for (BlockDocIdSet docIdSet : docIdSets) {
        if (docIdSet instanceof BitmapDocIdSet) {
            useBitmapOr = true;
        }
    }
    if (useBitmapOr) {
        List<ImmutableRoaringBitmap> allBitmaps = new ArrayList<ImmutableRoaringBitmap>();
        for (BlockDocIdSet docIdSet : docIdSets) {
            if (docIdSet instanceof SortedDocIdSet) {
                MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
                SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
                List<Pairs.IntPair> pairs = sortedDocIdSet.getRaw();
                for (Pairs.IntPair pair : pairs) {
                    //add takes [start, end) i.e inclusive start, exclusive end.
                    bitmap.add(pair.getLeft(), pair.getRight() + 1);
                }
                allBitmaps.add(bitmap);
            } else if (docIdSet instanceof BitmapDocIdSet) {
                BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
                ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
                allBitmaps.add(childBitmap);
            } else {
                BlockDocIdIterator iterator = docIdSet.iterator();
                rawIterators.add(iterator);
            }
        }
        MutableRoaringBitmap answer = allBitmaps.get(0).toMutableRoaringBitmap();
        for (int i = 1; i < allBitmaps.size(); i++) {
            answer.or(allBitmaps.get(i));
        }
        intIterator = answer.getIntIterator();
        BitmapDocIdIterator singleBitmapBlockIdIterator = new BitmapDocIdIterator(intIterator);
        singleBitmapBlockIdIterator.setStartDocId(minDocId);
        singleBitmapBlockIdIterator.setEndDocId(maxDocId);
        rawIterators.add(singleBitmapBlockIdIterator);
        docIdIterators = new BlockDocIdIterator[rawIterators.size()];
        rawIterators.toArray(docIdIterators);
    } else {
        docIdIterators = new BlockDocIdIterator[docIdSets.size()];
        for (int srcId = 0; srcId < docIdSets.size(); srcId++) {
            docIdIterators[srcId] = docIdSets.get(srcId).iterator();
        }
    }
    //    if (docIdIterators.length == 1) {
    //      return docIdIterators[0];
    //    } else {
    OrDocIdIterator orDocIdIterator = new OrDocIdIterator(docIdIterators);
    orDocIdIterator.setStartDocId(minDocId);
    orDocIdIterator.setEndDocId(maxDocId);
    return orDocIdIterator;
//    }
}
Also used : MutableRoaringBitmap(org.roaringbitmap.buffer.MutableRoaringBitmap) ArrayList(java.util.ArrayList) Pairs(com.linkedin.pinot.common.utils.Pairs) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) BlockDocIdSet(com.linkedin.pinot.core.common.BlockDocIdSet) ImmutableRoaringBitmap(org.roaringbitmap.buffer.ImmutableRoaringBitmap) BitmapDocIdIterator(com.linkedin.pinot.core.operator.dociditerators.BitmapDocIdIterator) OrDocIdIterator(com.linkedin.pinot.core.operator.dociditerators.OrDocIdIterator)

Example 28 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class OperatorSize method size.

public static int size(BlockDocIdSet docIdSet) {
    int count = 0;
    BlockDocIdIterator iterator = docIdSet.iterator();
    while (iterator.next() != Constants.EOF) {
        count++;
    }
    return count;
}
Also used : BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator)

Example 29 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test5.

@Test(enabled = false)
public void test5() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final List<String> rhs = new ArrayList<String>();
    rhs.add("-100");
    final Predicate p = new EqPredicate("column1", rhs);
    final DataSource ds = segment.getDataSource("column1", p);
    final Block bl = ds.nextBlock();
    final BlockDocIdSet idSet = bl.getBlockDocIdSet();
    final BlockDocIdIterator it = idSet.iterator();
    int docId = it.next();
    final StringBuilder b = new StringBuilder();
    while (docId != Constants.EOF) {
        b.append(docId + ",");
        docId = it.next();
    }
//    System.out.println(b.toString());
}
Also used : IndexSegmentImpl(com.linkedin.pinot.core.segment.index.IndexSegmentImpl) BlockDocIdSet(com.linkedin.pinot.core.common.BlockDocIdSet) ArrayList(java.util.ArrayList) Block(com.linkedin.pinot.core.common.Block) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) Predicate(com.linkedin.pinot.core.common.Predicate) DataSource(com.linkedin.pinot.core.common.DataSource) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test)

Example 30 with BlockDocIdIterator

use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.

the class ChunkIndexCreationDriverImplTest method test6.

@Test(enabled = false)
public void test6() throws Exception {
    final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
    final ImmutableDictionaryReader d = segment.getDictionaryFor("column7");
    final List<String> rhs = new ArrayList<String>();
    rhs.add(d.get(new Random().nextInt(d.length())).toString());
    final Predicate p = new EqPredicate("column7", rhs);
    final DataSource ds = segment.getDataSource("column7", p);
    final Block bl = ds.nextBlock();
    final BlockDocIdSet idSet = bl.getBlockDocIdSet();
    final BlockDocIdIterator it = idSet.iterator();
    int docId = it.next();
    final StringBuilder b = new StringBuilder();
    while (docId != Constants.EOF) {
        b.append(docId + ",");
        docId = it.next();
    }
//    System.out.println(b.toString());
}
Also used : ImmutableDictionaryReader(com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader) ArrayList(java.util.ArrayList) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) Predicate(com.linkedin.pinot.core.common.Predicate) DataSource(com.linkedin.pinot.core.common.DataSource) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) IndexSegmentImpl(com.linkedin.pinot.core.segment.index.IndexSegmentImpl) Random(java.util.Random) BlockDocIdSet(com.linkedin.pinot.core.common.BlockDocIdSet) Block(com.linkedin.pinot.core.common.Block) Test(org.testng.annotations.Test)

Aggregations

BlockDocIdIterator (com.linkedin.pinot.core.common.BlockDocIdIterator)33 ArrayList (java.util.ArrayList)28 Test (org.testng.annotations.Test)24 BlockDocIdSet (com.linkedin.pinot.core.common.BlockDocIdSet)14 Block (com.linkedin.pinot.core.common.Block)13 DataSource (com.linkedin.pinot.core.common.DataSource)11 Predicate (com.linkedin.pinot.core.common.Predicate)11 EqPredicate (com.linkedin.pinot.core.common.predicate.EqPredicate)11 NEqPredicate (com.linkedin.pinot.core.common.predicate.NEqPredicate)8 RangePredicate (com.linkedin.pinot.core.common.predicate.RangePredicate)8 BaseFilterBlock (com.linkedin.pinot.core.operator.blocks.BaseFilterBlock)8 BaseFilterOperator (com.linkedin.pinot.core.operator.filter.BaseFilterOperator)8 RealtimeSegmentImplTest (com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)8 IntPair (com.linkedin.pinot.common.utils.Pairs.IntPair)7 SortedDocIdSet (com.linkedin.pinot.core.operator.docidsets.SortedDocIdSet)5 MutableRoaringBitmap (org.roaringbitmap.buffer.MutableRoaringBitmap)5 BlockSingleValIterator (com.linkedin.pinot.core.common.BlockSingleValIterator)4 AndOperator (com.linkedin.pinot.core.operator.filter.AndOperator)4 BitmapBasedFilterOperator (com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator)4 OrOperator (com.linkedin.pinot.core.operator.filter.OrOperator)4