Examples with ImmutableRoaringBitmap org.roaringbitmap.buffer.ImmutableRoaringBitmap used on opensource projects

Example 16 with ImmutableRoaringBitmap

use of org.roaringbitmap.buffer.ImmutableRoaringBitmap in project pinot by linkedin.

the class BitmapPerformanceBenchmark method and.

private static ImmutableRoaringBitmap and(List<ImmutableRoaringBitmap> bitMaps) {
    if (bitMaps.size() == 1) {
        return bitMaps.get(0);
    }
    MutableRoaringBitmap answer = ImmutableRoaringBitmap.and(bitMaps.get(0), bitMaps.get(1));
    long start = System.currentTimeMillis();
    for (int i = 2; i < bitMaps.size(); i++) {
        answer.and(bitMaps.get(i));
    }
    long end = System.currentTimeMillis();
    int[] cardinality = new int[bitMaps.size()];
    int[] sizes = new int[bitMaps.size()];
    for (int i = 0; i < bitMaps.size(); i++) {
        ImmutableRoaringBitmap immutableRoaringBitmap = bitMaps.get(i);
        cardinality[i] = immutableRoaringBitmap.getCardinality();
        sizes[i] = immutableRoaringBitmap.getSizeInBytes();
    }
    bitMaps.get(0).getCardinality();
    System.out.println("AND operation Took " + (end - start));
    System.out.println("\toutput cardinality:" + answer.getCardinality());
    System.out.println("\toutout sizes:" + answer.getSizeInBytes());
    return answer;
}

Example 17 with ImmutableRoaringBitmap

use of org.roaringbitmap.buffer.ImmutableRoaringBitmap in project pinot by linkedin.

the class OrBlockDocIdSet method iterator.

@Override
public BlockDocIdIterator iterator() {
    List<BlockDocIdIterator> rawIterators = new ArrayList<>();
    boolean useBitmapOr = false;
    for (BlockDocIdSet docIdSet : docIdSets) {
        if (docIdSet instanceof BitmapDocIdSet) {
            useBitmapOr = true;
        }
    }
    if (useBitmapOr) {
        List<ImmutableRoaringBitmap> allBitmaps = new ArrayList<ImmutableRoaringBitmap>();
        for (BlockDocIdSet docIdSet : docIdSets) {
            if (docIdSet instanceof SortedDocIdSet) {
                MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
                SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
                List<Pairs.IntPair> pairs = sortedDocIdSet.getRaw();
                for (Pairs.IntPair pair : pairs) {
                    //add takes [start, end) i.e inclusive start, exclusive end.
                    bitmap.add(pair.getLeft(), pair.getRight() + 1);
                }
                allBitmaps.add(bitmap);
            } else if (docIdSet instanceof BitmapDocIdSet) {
                BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
                ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
                allBitmaps.add(childBitmap);
            } else {
                BlockDocIdIterator iterator = docIdSet.iterator();
                rawIterators.add(iterator);
            }
        }
        MutableRoaringBitmap answer = allBitmaps.get(0).toMutableRoaringBitmap();
        for (int i = 1; i < allBitmaps.size(); i++) {
            answer.or(allBitmaps.get(i));
        }
        intIterator = answer.getIntIterator();
        BitmapDocIdIterator singleBitmapBlockIdIterator = new BitmapDocIdIterator(intIterator);
        singleBitmapBlockIdIterator.setStartDocId(minDocId);
        singleBitmapBlockIdIterator.setEndDocId(maxDocId);
        rawIterators.add(singleBitmapBlockIdIterator);
        docIdIterators = new BlockDocIdIterator[rawIterators.size()];
        rawIterators.toArray(docIdIterators);
    } else {
        docIdIterators = new BlockDocIdIterator[docIdSets.size()];
        for (int srcId = 0; srcId < docIdSets.size(); srcId++) {
            docIdIterators[srcId] = docIdSets.get(srcId).iterator();
        }
    }
    //    if (docIdIterators.length == 1) {
    //      return docIdIterators[0];
    //    } else {
    OrDocIdIterator orDocIdIterator = new OrDocIdIterator(docIdIterators);
    orDocIdIterator.setStartDocId(minDocId);
    orDocIdIterator.setEndDocId(maxDocId);
    return orDocIdIterator;
//    }
}

Example 18 with ImmutableRoaringBitmap

use of org.roaringbitmap.buffer.ImmutableRoaringBitmap in project pinot by linkedin.

the class BitmapBasedFilterOperator method nextFilterBlock.

@Override
public BaseFilterBlock nextFilterBlock(BlockId BlockId) {
    InvertedIndexReader invertedIndex = dataSource.getInvertedIndex();
    Block dataSourceBlock = dataSource.nextBlock();
    int[] dictionaryIds;
    boolean exclusion = false;
    switch(predicate.getType()) {
        case EQ:
        case IN:
        case RANGE:
            dictionaryIds = predicateEvaluator.getMatchingDictionaryIds();
            break;
        case NEQ:
        case NOT_IN:
            exclusion = true;
            dictionaryIds = predicateEvaluator.getNonMatchingDictionaryIds();
            break;
        case REGEX:
        default:
            throw new UnsupportedOperationException("Regex is not supported");
    }
    ImmutableRoaringBitmap[] bitmaps = new ImmutableRoaringBitmap[dictionaryIds.length];
    for (int i = 0; i < dictionaryIds.length; i++) {
        bitmaps[i] = invertedIndex.getImmutable(dictionaryIds[i]);
    }
    bitmapBlock = new BitmapBlock(dataSource.getOperatorName(), dataSourceBlock.getMetadata(), startDocId, endDocId, bitmaps, exclusion);
    return bitmapBlock;
}