Examples with MutableRoaringBitmap org.roaringbitmap.buffer.MutableRoaringBitmap used on opensource projects

Example 21 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project pinot by linkedin.

the class SVScanDocIdIterator method applyAnd.

@Override
public MutableRoaringBitmap applyAnd(MutableRoaringBitmap answer) {
    MutableRoaringBitmap result = new MutableRoaringBitmap();
    if (evaluator.alwaysFalse()) {
        return result;
    }
    IntIterator intIterator = answer.getIntIterator();
    int docId = -1;
    while (intIterator.hasNext() && docId < endDocId) {
        docId = intIterator.next();
        if (docId >= startDocId) {
            valueIterator.skipTo(docId);
            _numEntriesScanned++;
            if (evaluator.apply(valueIterator.nextIntVal())) {
                result.add(docId);
            }
        }
    }
    return result;
}

Example 22 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project pinot by linkedin.

the class OrBlockDocIdSet method iterator.

@Override
public BlockDocIdIterator iterator() {
    List<BlockDocIdIterator> rawIterators = new ArrayList<>();
    boolean useBitmapOr = false;
    for (BlockDocIdSet docIdSet : docIdSets) {
        if (docIdSet instanceof BitmapDocIdSet) {
            useBitmapOr = true;
        }
    }
    if (useBitmapOr) {
        List<ImmutableRoaringBitmap> allBitmaps = new ArrayList<ImmutableRoaringBitmap>();
        for (BlockDocIdSet docIdSet : docIdSets) {
            if (docIdSet instanceof SortedDocIdSet) {
                MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
                SortedDocIdSet sortedDocIdSet = (SortedDocIdSet) docIdSet;
                List<Pairs.IntPair> pairs = sortedDocIdSet.getRaw();
                for (Pairs.IntPair pair : pairs) {
                    //add takes [start, end) i.e inclusive start, exclusive end.
                    bitmap.add(pair.getLeft(), pair.getRight() + 1);
                }
                allBitmaps.add(bitmap);
            } else if (docIdSet instanceof BitmapDocIdSet) {
                BitmapDocIdSet bitmapDocIdSet = (BitmapDocIdSet) docIdSet;
                ImmutableRoaringBitmap childBitmap = bitmapDocIdSet.getRaw();
                allBitmaps.add(childBitmap);
            } else {
                BlockDocIdIterator iterator = docIdSet.iterator();
                rawIterators.add(iterator);
            }
        }
        MutableRoaringBitmap answer = allBitmaps.get(0).toMutableRoaringBitmap();
        for (int i = 1; i < allBitmaps.size(); i++) {
            answer.or(allBitmaps.get(i));
        }
        intIterator = answer.getIntIterator();
        BitmapDocIdIterator singleBitmapBlockIdIterator = new BitmapDocIdIterator(intIterator);
        singleBitmapBlockIdIterator.setStartDocId(minDocId);
        singleBitmapBlockIdIterator.setEndDocId(maxDocId);
        rawIterators.add(singleBitmapBlockIdIterator);
        docIdIterators = new BlockDocIdIterator[rawIterators.size()];
        rawIterators.toArray(docIdIterators);
    } else {
        docIdIterators = new BlockDocIdIterator[docIdSets.size()];
        for (int srcId = 0; srcId < docIdSets.size(); srcId++) {
            docIdIterators[srcId] = docIdSets.get(srcId).iterator();
        }
    }
    //    if (docIdIterators.length == 1) {
    //      return docIdIterators[0];
    //    } else {
    OrDocIdIterator orDocIdIterator = new OrDocIdIterator(docIdIterators);
    orDocIdIterator.setStartDocId(minDocId);
    orDocIdIterator.setEndDocId(maxDocId);
    return orDocIdIterator;
//    }
}

Example 23 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project pinot by linkedin.

the class HeapBitmapInvertedIndexCreator method seal.

@Override
public void seal() throws IOException {
    final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(invertedIndexFile)));
    // First, write out offsets of bitmaps. The information can be used to access a certain bitmap directly.
    // Totally (invertedIndex.length+1) offsets will be written out; the last offset is used to calculate the length of
    // the last bitmap, which might be needed when accessing bitmaps randomly.
    // If a bitmap's offset is k, then k bytes need to be skipped to reach the bitmap.
    // The first bitmap's offset
    int offset = 4 * (invertedIndex.length + 1);
    out.writeInt(offset);
    for (final MutableRoaringBitmap element : invertedIndex) {
        // the other bitmap's offset
        offset += element.serializedSizeInBytes();
        out.writeInt(offset);
    }
    // write out bitmaps one by one
    for (final MutableRoaringBitmap element : invertedIndex) {
        element.serialize(out);
    }
    out.close();
    LOGGER.debug("persisted bitmap inverted index for column : " + spec.getName() + " in " + invertedIndexFile.getAbsolutePath());
}

Example 24 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project pinot by linkedin.

the class BitmapDocIdSetTest method testSimple.

@Test
public void testSimple() throws IOException {
    int numBitMaps = 5;
    final int numDocs = 1000;
    List<ImmutableRoaringBitmap> list = new ArrayList<ImmutableRoaringBitmap>();
    Random r = new Random();
    TreeSet<Integer> originalSet = new TreeSet<Integer>();
    for (int i = 0; i < numBitMaps; i++) {
        MutableRoaringBitmap mutableRoaringBitmap = new MutableRoaringBitmap();
        int length = r.nextInt(numDocs);
        for (int j = 0; j < length; j++) {
            int docId = r.nextInt(numDocs);
            originalSet.add(docId);
            mutableRoaringBitmap.add(docId);
        }
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(bos);
        // could call "rr1.runOptimize()" and "rr2.runOptimize" if there 
        // there were runs to compress
        mutableRoaringBitmap.serialize(dos);
        dos.close();
        ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
        ImmutableRoaringBitmap immutableRoaringBitmap = new ImmutableRoaringBitmap(bb);
        list.add(immutableRoaringBitmap);
    }
    ImmutableRoaringBitmap[] bitmaps = new ImmutableRoaringBitmap[list.size()];
    list.toArray(bitmaps);
    BlockMetadata blockMetadata = new BlockMetadata() {

        @Override
        public boolean isSparse() {
            // TODO Auto-generated method stub
            return false;
        }

        @Override
        public boolean isSorted() {
            // TODO Auto-generated method stub
            return false;
        }

        @Override
        public boolean isSingleValue() {
            // TODO Auto-generated method stub
            return false;
        }

        @Override
        public boolean hasInvertedIndex() {
            // TODO Auto-generated method stub
            return false;
        }

        @Override
        public boolean hasDictionary() {
            // TODO Auto-generated method stub
            return false;
        }

        @Override
        public int getStartDocId() {
            // TODO Auto-generated method stub
            return 0;
        }

        @Override
        public int getSize() {
            // TODO Auto-generated method stub
            return 0;
        }

        @Override
        public int getMaxNumberOfMultiValues() {
            // TODO Auto-generated method stub
            return 0;
        }

        @Override
        public int getLength() {
            return numDocs;
        }

        @Override
        public int getEndDocId() {
            return numDocs - 1;
        }

        @Override
        public Dictionary getDictionary() {
            // TODO Auto-generated method stub
            return null;
        }

        @Override
        public DataType getDataType() {
            // TODO Auto-generated method stub
            return null;
        }
    };
    BitmapDocIdSet bitmapDocIdSet = new BitmapDocIdSet("testColumn", blockMetadata, 0, numDocs - 1, bitmaps);
    BlockDocIdIterator iterator = bitmapDocIdSet.iterator();
    int docId;
    TreeSet<Integer> result = new TreeSet<Integer>();
    while ((docId = iterator.next()) != Constants.EOF) {
        result.add(docId);
    }
    Assert.assertEquals(originalSet.size(), result.size());
    Assert.assertEquals(originalSet, result);
}