Examples with MutableRoaringBitmap org.roaringbitmap.buffer.MutableRoaringBitmap used on opensource projects

Example 6 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project pinot by linkedin.

the class SortedInvertedIndexReader method getImmutable.

@Override
public ImmutableRoaringBitmap getImmutable(int idx) {
    if (idx >= cardinality) {
        return new MutableRoaringBitmap();
    }
    MutableRoaringBitmap rr = new MutableRoaringBitmap();
    int min = indexReader.getInt(idx, 0);
    int max = indexReader.getInt(idx, 1);
    for (int i = min; i <= max; i++) {
        rr.add(i);
    }
    return rr;
}

Example 7 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project pinot by linkedin.

the class BitmapPerformanceBenchmark method or.

private static ImmutableRoaringBitmap or(List<ImmutableRoaringBitmap> bitMaps) {
    if (bitMaps.size() == 1) {
        return bitMaps.get(0);
    }
    MutableRoaringBitmap answer = ImmutableRoaringBitmap.or(bitMaps.get(0), bitMaps.get(1));
    long start = System.currentTimeMillis();
    for (int i = 2; i < bitMaps.size(); i++) {
        answer.or(bitMaps.get(i));
    }
    long end = System.currentTimeMillis();
    bitMaps.get(0).getCardinality();
    System.out.println("OR operation Took " + (end - start));
    System.out.println("\toutput cardinality:" + answer.getCardinality());
    System.out.println("\toutout sizes:" + answer.getSizeInBytes());
    return answer;
}

Example 8 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project pinot by linkedin.

the class ForwardIndexWriterBenchmark method convertRawToForwardIndex.

public static void convertRawToForwardIndex(File rawFile) throws Exception {
    List<String> lines = IOUtils.readLines(new FileReader(rawFile));
    int totalDocs = lines.size();
    int max = Integer.MIN_VALUE;
    int maxNumberOfMultiValues = Integer.MIN_VALUE;
    int totalNumValues = 0;
    int[][] data = new int[totalDocs][];
    for (int i = 0; i < lines.size(); i++) {
        String line = lines.get(i);
        String[] split = line.split(",");
        totalNumValues = totalNumValues + split.length;
        if (split.length > maxNumberOfMultiValues) {
            maxNumberOfMultiValues = split.length;
        }
        data[i] = new int[split.length];
        for (int j = 0; j < split.length; j++) {
            String token = split[j];
            int val = Integer.parseInt(token);
            data[i][j] = val;
            if (val > max) {
                max = val;
            }
        }
    }
    int maxBitsNeeded = (int) Math.ceil(Math.log(max) / Math.log(2));
    int size = 2048;
    int[] offsets = new int[size];
    int bitMapSize = 0;
    File outputFile = new File("output.mv.fwd");
    FixedBitMultiValueWriter fixedBitSkipListSCMVWriter = new FixedBitMultiValueWriter(outputFile, totalDocs, totalNumValues, maxBitsNeeded);
    for (int i = 0; i < totalDocs; i++) {
        fixedBitSkipListSCMVWriter.setIntArray(i, data[i]);
        if (i % size == size - 1) {
            MutableRoaringBitmap rr1 = MutableRoaringBitmap.bitmapOf(offsets);
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            DataOutputStream dos = new DataOutputStream(bos);
            rr1.serialize(dos);
            dos.close();
            // System.out.println("Chunk " + i / size + " bitmap size:" + bos.size());
            bitMapSize += bos.size();
        } else if (i == totalDocs - 1) {
            MutableRoaringBitmap rr1 = MutableRoaringBitmap.bitmapOf(Arrays.copyOf(offsets, i % size));
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            DataOutputStream dos = new DataOutputStream(bos);
            rr1.serialize(dos);
            dos.close();
            // System.out.println("Chunk " + i / size + " bitmap size:" + bos.size());
            bitMapSize += bos.size();
        }
    }
    fixedBitSkipListSCMVWriter.close();
    System.out.println("Output file size:" + outputFile.length());
    System.out.println("totalNumberOfDoc\t\t\t:" + totalDocs);
    System.out.println("totalNumberOfValues\t\t\t:" + totalNumValues);
    System.out.println("chunk size\t\t\t\t:" + size);
    System.out.println("Num chunks\t\t\t\t:" + totalDocs / size);
    int numChunks = totalDocs / size + 1;
    int totalBits = (totalNumValues * maxBitsNeeded);
    int dataSizeinBytes = (totalBits + 7) / 8;
    System.out.println("Raw data size with fixed bit encoding\t:" + dataSizeinBytes);
    System.out.println("\nPer encoding size");
    System.out.println();
    System.out.println("size (offset + length)\t\t\t:" + ((totalDocs * (4 + 4)) + dataSizeinBytes));
    System.out.println();
    System.out.println("size (offset only)\t\t\t:" + ((totalDocs * (4)) + dataSizeinBytes));
    System.out.println();
    System.out.println("bitMapSize\t\t\t\t:" + bitMapSize);
    System.out.println("size (with bitmap)\t\t\t:" + (bitMapSize + (numChunks * 4) + dataSizeinBytes));
    System.out.println();
    System.out.println("Custom Bitset\t\t\t\t:" + (totalNumValues + 7) / 8);
    System.out.println("size (with custom bitset)\t\t\t:" + (((totalNumValues + 7) / 8) + (numChunks * 4) + dataSizeinBytes));
}

Example 9 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project druid by druid-io.

the class RangeBitmapBenchmarkTest method prepareRandomRanges.

@BeforeClass
public static void prepareRandomRanges() throws Exception {
    System.setProperty("jub.customkey", String.format("%06.5f", DENSITY));
    reset();
    final BitSet expectedUnion = new BitSet();
    for (int i = 0; i < SIZE; ++i) {
        ConciseSet c = new ConciseSet();
        MutableRoaringBitmap r = new MutableRoaringBitmap();
        {
            int k = 0;
            boolean fill = true;
            while (k < LENGTH) {
                int runLength = (int) (LENGTH * DENSITY) + rand.nextInt((int) (LENGTH * DENSITY));
                for (int j = k; fill && j < LENGTH && j < k + runLength; ++j) {
                    c.add(j);
                    r.add(j);
                    expectedUnion.set(j);
                }
                k += runLength;
                fill = !fill;
            }
        }
        minIntersection = MIN_INTERSECT;
        for (int k = LENGTH / 2; k < LENGTH / 2 + minIntersection; ++k) {
            c.add(k);
            r.add(k);
            expectedUnion.set(k);
        }
        concise[i] = ImmutableConciseSet.newImmutableFromMutable(c);
        offheapConcise[i] = makeOffheapConcise(concise[i]);
        roaring[i] = r;
        immutableRoaring[i] = makeImmutableRoaring(r);
        offheapRoaring[i] = makeOffheapRoaring(r);
        genericConcise[i] = new WrappedImmutableConciseBitmap(offheapConcise[i]);
        genericRoaring[i] = new WrappedImmutableRoaringBitmap(offheapRoaring[i]);
    }
    unionCount = expectedUnion.cardinality();
    printSizeStats(DENSITY, "Random Alternating Bitmap");
}

Example 10 with MutableRoaringBitmap

use of org.roaringbitmap.buffer.MutableRoaringBitmap in project druid by druid-io.

the class UniformBitmapBenchmarkTest method prepareMostlyUniform.

@BeforeClass
public static void prepareMostlyUniform() throws Exception {
    System.setProperty("jub.customkey", String.format("%05.4f", DENSITY));
    reset();
    final BitSet expectedUnion = new BitSet();
    final int[] knownTrue = new int[MIN_INTERSECT];
    for (int i = 0; i < knownTrue.length; ++i) {
        knownTrue[i] = rand.nextInt(LENGTH);
    }
    for (int i = 0; i < SIZE; ++i) {
        ConciseSet c = new ConciseSet();
        MutableRoaringBitmap r = new MutableRoaringBitmap();
        for (int k = 0; k < LENGTH; ++k) {
            if (rand.nextDouble() < DENSITY) {
                c.add(k);
                r.add(k);
                expectedUnion.set(k);
            }
        }
        for (int k : knownTrue) {
            c.add(k);
            r.add(k);
            expectedUnion.set(k);
        }
        concise[i] = ImmutableConciseSet.newImmutableFromMutable(c);
        offheapConcise[i] = makeOffheapConcise(concise[i]);
        roaring[i] = r;
        immutableRoaring[i] = makeImmutableRoaring(r);
        offheapRoaring[i] = makeOffheapRoaring(r);
        genericConcise[i] = new WrappedImmutableConciseBitmap(offheapConcise[i]);
        genericRoaring[i] = new WrappedImmutableRoaringBitmap(offheapRoaring[i]);
    }
    unionCount = expectedUnion.cardinality();
    minIntersection = knownTrue.length;
    printSizeStats(DENSITY, "Uniform Bitmap");
}