use of org.roaringbitmap.buffer.ImmutableRoaringBitmap in project druid by druid-io.
the class WrappedImmutableRoaringBitmap method intersection.
@Override
public ImmutableBitmap intersection(ImmutableBitmap otherBitmap) {
WrappedImmutableRoaringBitmap other = (WrappedImmutableRoaringBitmap) otherBitmap;
ImmutableRoaringBitmap unwrappedOtherBitmap = other.bitmap;
return new WrappedImmutableRoaringBitmap(ImmutableRoaringBitmap.and(bitmap, unwrappedOtherBitmap));
}
use of org.roaringbitmap.buffer.ImmutableRoaringBitmap in project pinot by linkedin.
the class AndBlockDocIdSet method slowIterator.
public BlockDocIdIterator slowIterator() {
List<BlockDocIdIterator> rawIterators = new ArrayList<>();
boolean useBitmapBasedIntersection = false;
for (BlockDocIdSet docIdSet : blockDocIdSets) {
if (docIdSet instanceof BitmapDocIdSet) {
useBitmapBasedIntersection = true;
}
}
BlockDocIdIterator[] docIdIterators;
if (useBitmapBasedIntersection) {
List<ImmutableRoaringBitmap> allBitmaps = new ArrayList<ImmutableRoaringBitmap>();
for (BlockDocIdSet docIdSet : blockDocIdSets) {
if (docIdSet instanceof SortedDocIdSet) {
MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
List<IntPair> pairs = sortedDocIdSet.getRaw();
for (IntPair pair : pairs) {
// add takes [start, end) i.e inclusive
bitmap.add(pair.getLeft(), pair.getRight() + 1);
// start, exclusive end.
}
allBitmaps.add(bitmap);
} else if (docIdSet instanceof BitmapDocIdSet) {
BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
allBitmaps.add(childBitmap);
} else {
BlockDocIdIterator iterator = docIdSet.iterator();
rawIterators.add(iterator);
}
}
IntIterator intIterator;
if (allBitmaps.size() > 1) {
MutableRoaringBitmap answer = (MutableRoaringBitmap) allBitmaps.get(0).clone();
for (int i = 1; i < allBitmaps.size(); i++) {
answer.and(allBitmaps.get(i));
}
intIterator = answer.getIntIterator();
} else {
intIterator = allBitmaps.get(0).getIntIterator();
}
BitmapDocIdIterator singleBitmapBlockIdIterator = new BitmapDocIdIterator(intIterator);
singleBitmapBlockIdIterator.setStartDocId(minDocId);
singleBitmapBlockIdIterator.setEndDocId(maxDocId);
rawIterators.add(0, singleBitmapBlockIdIterator);
docIdIterators = new BlockDocIdIterator[rawIterators.size()];
rawIterators.toArray(docIdIterators);
} else {
docIdIterators = new BlockDocIdIterator[blockDocIdSets.size()];
for (int srcId = 0; srcId < blockDocIdSets.size(); srcId++) {
docIdIterators[srcId] = blockDocIdSets.get(srcId).iterator();
}
}
return new AndDocIdIterator(docIdIterators);
}
use of org.roaringbitmap.buffer.ImmutableRoaringBitmap in project pinot by linkedin.
the class AndBlockDocIdSet method fastIterator.
public BlockDocIdIterator fastIterator() {
long start = System.currentTimeMillis();
List<List<IntPair>> sortedRangeSets = new ArrayList<>();
List<ImmutableRoaringBitmap> childBitmaps = new ArrayList<ImmutableRoaringBitmap>();
List<FilterBlockDocIdSet> scanBasedDocIdSets = new ArrayList<>();
List<BlockDocIdIterator> remainingIterators = new ArrayList<>();
for (BlockDocIdSet docIdSet : blockDocIdSets) {
if (docIdSet instanceof SortedDocIdSet) {
SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
List<IntPair> pairs = sortedDocIdSet.getRaw();
sortedRangeSets.add(pairs);
} else if (docIdSet instanceof BitmapDocIdSet) {
BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
childBitmaps.add(childBitmap);
} else if (docIdSet instanceof ScanBasedSingleValueDocIdSet) {
scanBasedDocIdSets.add((ScanBasedSingleValueDocIdSet) docIdSet);
} else if (docIdSet instanceof ScanBasedMultiValueDocIdSet) {
scanBasedDocIdSets.add((ScanBasedMultiValueDocIdSet) docIdSet);
} else {
// TODO:handle child OR/AND as bitmap if possible
remainingIterators.add(docIdSet.iterator());
}
}
if (childBitmaps.size() == 0 && sortedRangeSets.size() == 0) {
// When one or more of the operands are operators themselves, then we don't have a sorted or
// bitmap index. In that case, just use the AndDocIdIterator to iterate over all of of the subtree.
BlockDocIdIterator[] docIdIterators = new BlockDocIdIterator[blockDocIdSets.size()];
for (int srcId = 0; srcId < blockDocIdSets.size(); srcId++) {
docIdIterators[srcId] = blockDocIdSets.get(srcId).iterator();
}
return new AndDocIdIterator(docIdIterators);
} else {
// TODO: will be nice to re-order sorted and bitmap index based on size
if (sortedRangeSets.size() > 0) {
List<IntPair> pairList;
pairList = SortedRangeIntersection.intersectSortedRangeSets(sortedRangeSets);
answer = new MutableRoaringBitmap();
for (IntPair pair : pairList) {
// end is exclusive
answer.add(pair.getLeft(), pair.getRight() + 1);
}
}
// handle bitmaps
if (childBitmaps.size() > 0) {
if (answer == null) {
answer = childBitmaps.get(0).toMutableRoaringBitmap();
for (int i = 1; i < childBitmaps.size(); i++) {
answer.and(childBitmaps.get(i));
}
} else {
for (int i = 0; i < childBitmaps.size(); i++) {
answer.and(childBitmaps.get(i));
}
}
}
// At this point, we must have 'answer' to be non-null.
assert (answer != null) : "sortedRangeSets=" + sortedRangeSets.size() + ",childBitmaps=" + childBitmaps.size();
// handle raw iterators
for (FilterBlockDocIdSet scanBasedDocIdSet : scanBasedDocIdSets) {
ScanBasedDocIdIterator iterator = (ScanBasedDocIdIterator) scanBasedDocIdSet.iterator();
MutableRoaringBitmap scanAnswer = iterator.applyAnd(answer);
answer.and(scanAnswer);
}
long end = System.currentTimeMillis();
LOGGER.debug("Time to evaluate and Filter:{}", (end - start));
// if other iterators exists resort to iterator style intersection
BlockDocIdIterator answerDocIdIterator = new RangelessBitmapDocIdIterator(answer.getIntIterator());
if (remainingIterators.size() == 0) {
return answerDocIdIterator;
} else {
BlockDocIdIterator[] docIdIterators = new BlockDocIdIterator[remainingIterators.size() + 1];
docIdIterators[0] = answerDocIdIterator;
for (int i = 0; i < remainingIterators.size(); i++) {
docIdIterators[i + 1] = remainingIterators.get(i);
}
return new AndDocIdIterator(docIdIterators);
}
}
}
use of org.roaringbitmap.buffer.ImmutableRoaringBitmap in project pinot by linkedin.
the class BitmapInvertedIndexCreatorTest method validate.
private void validate(String colName, File bitmapIndexFile, int cardinality, Map<Integer, Set<Integer>> postingListMap) throws IOException {
Assert.assertTrue(bitmapIndexFile.exists());
PinotDataBuffer dataBuffer = PinotDataBuffer.fromFile(bitmapIndexFile, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
BitmapInvertedIndexReader reader = new BitmapInvertedIndexReader(dataBuffer, cardinality);
for (int i = 0; i < cardinality; i++) {
ImmutableRoaringBitmap bitmap = reader.getImmutable(i);
Set<Integer> expected = postingListMap.get(i);
Assert.assertEquals(bitmap.getCardinality(), expected.size());
int[] actual = bitmap.toArray();
List<Integer> actualList = Ints.asList(actual);
Assert.assertEquals(actualList, expected);
}
dataBuffer.close();
}
use of org.roaringbitmap.buffer.ImmutableRoaringBitmap in project pinot by linkedin.
the class BitmapPerformanceBenchmark method iterationSpeed.
public static void iterationSpeed(String indexSegmentDir, String column) throws Exception {
File indexSegment = new File(indexSegmentDir);
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexSegment);
Map<String, BitmapInvertedIndexReader> bitMapIndexMap = new HashMap<String, BitmapInvertedIndexReader>();
Map<String, Integer> cardinalityMap = new HashMap<String, Integer>();
Map<String, ImmutableDictionaryReader> dictionaryMap = new HashMap<String, ImmutableDictionaryReader>();
File bitMapIndexFile = new File(indexSegmentDir, column + ".bitmap.inv");
ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column);
int cardinality = columnMetadata.getCardinality();
cardinalityMap.put(column, cardinality);
PinotDataBuffer bitMapDataBuffer = PinotDataBuffer.fromFile(bitMapIndexFile, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
BitmapInvertedIndexReader bitmapInvertedIndex = new BitmapInvertedIndexReader(bitMapDataBuffer, cardinality);
File dictionaryFile = new File(indexSegmentDir + "/" + column + ".dict");
SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexSegment, segmentMetadata, ReadMode.mmap);
SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
ColumnIndexContainer container = ColumnIndexContainer.init(segmentReader, columnMetadata, null);
ImmutableDictionaryReader dictionary = container.getDictionary();
dictionaryMap.put(column, dictionary);
// System.out.println(column + ":\t" + MemoryUtil.deepMemoryUsageOf(bitmapInvertedIndex));
bitMapIndexMap.put(column, bitmapInvertedIndex);
int dictId = dictionary.indexOf("na.us");
ImmutableRoaringBitmap immutable = bitmapInvertedIndex.getImmutable(dictId);
Iterator<Integer> iterator = immutable.iterator();
int count = 0;
long start = System.currentTimeMillis();
while (iterator.hasNext()) {
iterator.next();
count = count + 1;
}
long end = System.currentTimeMillis();
System.out.println(" matched: " + count + " Time to iterate:" + (end - start));
bitMapDataBuffer.close();
}
Aggregations