use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.
the class AndBlockDocIdSet method iterator.
@Override
public BlockDocIdIterator iterator() {
//TODO: Remove this validation code once we have enough testing
if (validate) {
BlockDocIdIterator slowIterator = slowIterator();
BlockDocIdIterator fastIterator = fastIterator();
List<Integer> matchedIds = new ArrayList<>();
while (true) {
int docId1 = slowIterator.next();
int docId2 = fastIterator.next();
if (docId1 != docId2) {
LOGGER.error("ERROR docId1:" + docId1 + " docId2:" + docId2);
} else {
matchedIds.add(docId1);
}
if (docId1 == Constants.EOF || docId2 == Constants.EOF) {
break;
}
}
answer = null;
}
return fastIterator();
}
use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.
the class OrBlockDocIdSet method iterator.
@Override
public BlockDocIdIterator iterator() {
List<BlockDocIdIterator> rawIterators = new ArrayList<>();
boolean useBitmapOr = false;
for (BlockDocIdSet docIdSet : docIdSets) {
if (docIdSet instanceof BitmapDocIdSet) {
useBitmapOr = true;
}
}
if (useBitmapOr) {
List<ImmutableRoaringBitmap> allBitmaps = new ArrayList<ImmutableRoaringBitmap>();
for (BlockDocIdSet docIdSet : docIdSets) {
if (docIdSet instanceof SortedDocIdSet) {
MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
List<Pairs.IntPair> pairs = sortedDocIdSet.getRaw();
for (Pairs.IntPair pair : pairs) {
//add takes [start, end) i.e inclusive start, exclusive end.
bitmap.add(pair.getLeft(), pair.getRight() + 1);
}
allBitmaps.add(bitmap);
} else if (docIdSet instanceof BitmapDocIdSet) {
BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
allBitmaps.add(childBitmap);
} else {
BlockDocIdIterator iterator = docIdSet.iterator();
rawIterators.add(iterator);
}
}
MutableRoaringBitmap answer = allBitmaps.get(0).toMutableRoaringBitmap();
for (int i = 1; i < allBitmaps.size(); i++) {
answer.or(allBitmaps.get(i));
}
intIterator = answer.getIntIterator();
BitmapDocIdIterator singleBitmapBlockIdIterator = new BitmapDocIdIterator(intIterator);
singleBitmapBlockIdIterator.setStartDocId(minDocId);
singleBitmapBlockIdIterator.setEndDocId(maxDocId);
rawIterators.add(singleBitmapBlockIdIterator);
docIdIterators = new BlockDocIdIterator[rawIterators.size()];
rawIterators.toArray(docIdIterators);
} else {
docIdIterators = new BlockDocIdIterator[docIdSets.size()];
for (int srcId = 0; srcId < docIdSets.size(); srcId++) {
docIdIterators[srcId] = docIdSets.get(srcId).iterator();
}
}
// if (docIdIterators.length == 1) {
// return docIdIterators[0];
// } else {
OrDocIdIterator orDocIdIterator = new OrDocIdIterator(docIdIterators);
orDocIdIterator.setStartDocId(minDocId);
orDocIdIterator.setEndDocId(maxDocId);
return orDocIdIterator;
// }
}
use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.
the class OperatorSize method size.
public static int size(BlockDocIdSet docIdSet) {
int count = 0;
BlockDocIdIterator iterator = docIdSet.iterator();
while (iterator.next() != Constants.EOF) {
count++;
}
return count;
}
use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.
the class ChunkIndexCreationDriverImplTest method test5.
@Test(enabled = false)
public void test5() throws Exception {
final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
final List<String> rhs = new ArrayList<String>();
rhs.add("-100");
final Predicate p = new EqPredicate("column1", rhs);
final DataSource ds = segment.getDataSource("column1", p);
final Block bl = ds.nextBlock();
final BlockDocIdSet idSet = bl.getBlockDocIdSet();
final BlockDocIdIterator it = idSet.iterator();
int docId = it.next();
final StringBuilder b = new StringBuilder();
while (docId != Constants.EOF) {
b.append(docId + ",");
docId = it.next();
}
// System.out.println(b.toString());
}
use of com.linkedin.pinot.core.common.BlockDocIdIterator in project pinot by linkedin.
the class ChunkIndexCreationDriverImplTest method test6.
@Test(enabled = false)
public void test6() throws Exception {
final IndexSegmentImpl segment = (IndexSegmentImpl) Loaders.IndexSegment.load(INDEX_DIR.listFiles()[0], ReadMode.mmap);
final ImmutableDictionaryReader d = segment.getDictionaryFor("column7");
final List<String> rhs = new ArrayList<String>();
rhs.add(d.get(new Random().nextInt(d.length())).toString());
final Predicate p = new EqPredicate("column7", rhs);
final DataSource ds = segment.getDataSource("column7", p);
final Block bl = ds.nextBlock();
final BlockDocIdSet idSet = bl.getBlockDocIdSet();
final BlockDocIdIterator it = idSet.iterator();
int docId = it.next();
final StringBuilder b = new StringBuilder();
while (docId != Constants.EOF) {
b.append(docId + ",");
docId = it.next();
}
// System.out.println(b.toString());
}
Aggregations