Examples with CustomBitSet com.linkedin.pinot.core.util.CustomBitSet used on opensource projects

Example 1 with CustomBitSet

use of com.linkedin.pinot.core.util.CustomBitSet in project pinot by linkedin.

the class FixedBitWidthRowColDataFileReaderTest method testReadIntFromByteBuffer.

@Test
public void testReadIntFromByteBuffer() {
    int maxBits = 1;
    while (maxBits < 32) {
        //      System.out.println("START MAX BITS:" + maxBits);
        int numElements = 100;
        CustomBitSet customBitSet = CustomBitSet.withBitLength(numElements * maxBits);
        int max = (int) Math.pow(2, maxBits);
        Random r = new Random();
        int[] values = new int[numElements];
        for (int i = 0; i < numElements; i++) {
            int value = r.nextInt(max);
            values[i] = value;
            LOGGER.info(Integer.toString(value));
            StringBuilder sb = new StringBuilder();
            for (int j = maxBits - 1; j >= 0; j--) {
                if ((value & (1 << j)) != 0) {
                    sb.append("1");
                    customBitSet.setBit(i * maxBits + (maxBits - j - 1));
                } else {
                    sb.append("0");
                }
            }
            LOGGER.info(sb.toString());
        }
        LOGGER.info("customBitSet: " + customBitSet.toString());
        int bitPos = 0;
        for (int i = 0; i < numElements; i++) {
            bitPos = i * maxBits;
            int readInt = customBitSet.readInt(bitPos, bitPos + maxBits);
            if (readInt != values[i]) {
                readInt = customBitSet.readInt(bitPos, bitPos + maxBits);
            }
            Assert.assertEquals(readInt, values[i]);
        }
        //      System.out.println("END MAX BITS:" + maxBits);
        maxBits = maxBits + 1;
        customBitSet.close();
    }
}

Example 2 with CustomBitSet

use of com.linkedin.pinot.core.util.CustomBitSet in project pinot by linkedin.

the class FixedBitWidthRowColDataFileReaderTest method testSingleColUnsigned.

/**
   * Tests only positive numbers
   *
   * @throws Exception
   */
@Test
public void testSingleColUnsigned() throws Exception {
    int[] maxBitArray = new int[] { 4 };
    for (int maxBits : maxBitArray) {
        String fileName = "test" + maxBits + "FixedBitWidthSingleCol";
        File file = new File(fileName);
        CustomBitSet bitset = null;
        try {
            //        System.out.println("START MAX BITS:" + maxBits);
            int numElements = 100;
            bitset = CustomBitSet.withBitLength(numElements * maxBits);
            int max = (int) Math.pow(2, maxBits);
            Random r = new Random();
            int[] values = new int[numElements];
            for (int i = 0; i < numElements; i++) {
                int value = r.nextInt(max);
                values[i] = value;
                for (int j = maxBits - 1; j >= 0; j--) {
                    if ((value & (1 << j)) != 0) {
                        bitset.setBit(i * maxBits + (maxBits - j - 1));
                    }
                }
            }
            byte[] byteArray = bitset.toByteArray();
            FileOutputStream fos = new FileOutputStream(file);
            fos.write(byteArray);
            fos.close();
            PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
            FixedBitSingleValueMultiColReader heapReader = new FixedBitSingleValueMultiColReader(heapBuffer, numElements, 1, new int[] { maxBits });
            for (int i = 0; i < numElements; i++) {
                int readInt = heapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            heapReader.close();
            heapBuffer.close();
            PinotDataBuffer dataBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap-testing");
            FixedBitSingleValueMultiColReader mmapReader = new FixedBitSingleValueMultiColReader(dataBuffer, numElements, 1, new int[] { maxBits });
            for (int i = 0; i < numElements; i++) {
                int readInt = mmapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            mmapReader.close();
            dataBuffer.close();
        //        System.out.println("END MAX BITS:" + maxBits);
        } finally {
            file.delete();
            bitset.close();
        }
    }
}

Example 3 with CustomBitSet

use of com.linkedin.pinot.core.util.CustomBitSet in project pinot by linkedin.

the class FixedByteSkipListSCMVWriterTest method testSingleColMultiValue.

@Test
public void testSingleColMultiValue() throws Exception {
    File file = new File("test_single_col_multi_value_writer.dat");
    file.delete();
    int rows = 100;
    int[][] data = new int[rows][];
    Random r = new Random();
    int totalNumValues = 0;
    for (int i = 0; i < rows; i++) {
        int numValues = r.nextInt(100) + 1;
        data[i] = new int[numValues];
        for (int j = 0; j < numValues; j++) {
            data[i][j] = r.nextInt();
        }
        totalNumValues += numValues;
    }
    FixedByteSkipListMultiValueWriter writer = new FixedByteSkipListMultiValueWriter(file, rows, totalNumValues, 4);
    int numChunks = writer.getNumChunks();
    int[] chunkOffsets = new int[numChunks];
    int chunkId = 0;
    int offset = 0;
    for (int i = 0; i < rows; i++) {
        writer.setIntArray(i, data[i]);
        if (i % writer.getDocsPerChunk() == 0) {
            chunkOffsets[chunkId] = offset;
            chunkId = chunkId + 1;
        }
        offset += data[i].length;
    }
    writer.close();
    DataInputStream dis = new DataInputStream(new FileInputStream(file));
    for (int i = 0; i < numChunks; i++) {
        Assert.assertEquals(dis.readInt(), chunkOffsets[i]);
    }
    int numBytesForBitmap = (totalNumValues + 7) / 8;
    byte[] bitsetBytes = new byte[numBytesForBitmap];
    dis.read(bitsetBytes);
    CustomBitSet customBit = CustomBitSet.withByteBuffer(numBytesForBitmap, ByteBuffer.wrap(bitsetBytes));
    offset = 0;
    //    System.out.println("getChunkOffsetHeaderSize:" + writer.getChunkOffsetHeaderSize());
    for (int i = 0; i < rows; i++) {
        Assert.assertTrue(customBit.isBitSet(offset));
        for (int j = 0; j < data[i].length; j++) {
            Assert.assertEquals(dis.readInt(), data[i][j]);
        }
        offset += data[i].length;
    }
    dis.close();
    file.delete();
    //    raf.close();
    customBit.close();
}

Example 4 with CustomBitSet

use of com.linkedin.pinot.core.util.CustomBitSet in project pinot by linkedin.

the class FixedBitWidthRowColDataFileReaderTest method testSingleColSigned.

/**
   * Tests both positive and negative numbers
   *
   * @throws Exception
   */
@Test
public void testSingleColSigned() throws Exception {
    int[] maxBitArray = new int[] { 4 };
    for (int maxBits : maxBitArray) {
        String fileName = "test" + maxBits + "FixedBitWidthSingleCol";
        File file = new File(fileName);
        CustomBitSet bitset = null;
        try {
            //        System.out.println("START MAX BITS:" + maxBits);
            int numElements = 100;
            int requiredBits = maxBits + 1;
            bitset = CustomBitSet.withBitLength(numElements * requiredBits);
            int max = (int) Math.pow(2, maxBits);
            Random r = new Random();
            int[] values = new int[numElements];
            int offset = max - 1;
            for (int i = 0; i < numElements; i++) {
                int value = r.nextInt(max);
                if (Math.random() > .5) {
                    value = -1 * value;
                }
                values[i] = value;
                int offsetValue = offset + value;
                for (int j = requiredBits - 1; j >= 0; j--) {
                    if ((offsetValue & (1 << j)) != 0) {
                        bitset.setBit(i * requiredBits + (requiredBits - j - 1));
                    }
                }
            }
            byte[] byteArray = bitset.toByteArray();
            FileOutputStream fos = new FileOutputStream(file);
            fos.write(byteArray);
            fos.close();
            PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
            FixedBitSingleValueMultiColReader heapReader = new FixedBitSingleValueMultiColReader(heapBuffer, numElements, 1, new int[] { maxBits }, new boolean[] { true });
            for (int i = 0; i < numElements; i++) {
                int readInt = heapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            heapReader.close();
            heapBuffer.close();
            PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap-testing");
            FixedBitSingleValueMultiColReader mmapReader = new FixedBitSingleValueMultiColReader(mmapBuffer, numElements, 1, new int[] { maxBits }, new boolean[] { true });
            for (int i = 0; i < numElements; i++) {
                int readInt = mmapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            mmapReader.close();
            mmapBuffer.close();
        //        System.out.println("END MAX BITS:" + maxBits);
        } finally {
            file.delete();
            bitset.close();
        }
    }
}

Example 5 with CustomBitSet

use of com.linkedin.pinot.core.util.CustomBitSet in project pinot by linkedin.

the class FixedBitSkipListSCMVWriterTest method testSingleColMultiValue.

@Test
public void testSingleColMultiValue() throws Exception {
    int maxBits = 2;
    while (maxBits < 32) {
        LOGGER.debug("START test maxBit:" + maxBits);
        File file = new File("test_single_col_multi_value_writer.dat");
        file.delete();
        int rows = 100;
        int[][] data = new int[rows][];
        int maxValue = (int) Math.pow(2, maxBits);
        Random r = new Random();
        int totalNumValues = 0;
        for (int i = 0; i < rows; i++) {
            int numValues = r.nextInt(100) + 1;
            data[i] = new int[numValues];
            for (int j = 0; j < numValues; j++) {
                data[i][j] = r.nextInt(maxValue);
            }
            totalNumValues += numValues;
        }
        FixedBitMultiValueWriter writer = new FixedBitMultiValueWriter(file, rows, totalNumValues, maxBits);
        CustomBitSet bitSet = CustomBitSet.withBitLength(totalNumValues * maxBits);
        int numChunks = writer.getNumChunks();
        int[] chunkOffsets = new int[numChunks];
        int chunkId = 0;
        int offset = 0;
        int index = 0;
        for (int i = 0; i < rows; i++) {
            writer.setIntArray(i, data[i]);
            if (i % writer.getRowsPerChunk() == 0) {
                chunkOffsets[chunkId] = offset;
                chunkId = chunkId + 1;
            }
            offset += data[i].length;
            for (int j = 0; j < data[i].length; j++) {
                int value = data[i][j];
                for (int bitPos = maxBits - 1; bitPos >= 0; bitPos--) {
                    if ((value & (1 << bitPos)) != 0) {
                        bitSet.setBit(index * maxBits + (maxBits - bitPos - 1));
                    }
                }
                index = index + 1;
            }
        }
        writer.close();
        LOGGER.trace("chunkOffsets: {}", Arrays.toString(chunkOffsets));
        //start validating the file
        RandomAccessFile raf = new RandomAccessFile(file, "r");
        Assert.assertEquals(raf.length(), writer.getTotalSize());
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        for (int i = 0; i < numChunks; i++) {
            Assert.assertEquals(dis.readInt(), chunkOffsets[i]);
        }
        int numBytesForBitmap = (totalNumValues + 7) / 8;
        Assert.assertEquals(writer.getBitsetSize(), numBytesForBitmap);
        byte[] bitsetBytes = new byte[numBytesForBitmap];
        dis.read(bitsetBytes);
        CustomBitSet customBit = CustomBitSet.withByteBuffer(numBytesForBitmap, ByteBuffer.wrap(bitsetBytes));
        offset = 0;
        LOGGER.trace(customBit.toString());
        for (int i = 0; i < rows; i++) {
            Assert.assertTrue(customBit.isBitSet(offset));
            offset += data[i].length;
        }
        byte[] byteArray = bitSet.toByteArray();
        LOGGER.trace("raf.length():" + raf.length());
        LOGGER.trace("getTotalSize:" + writer.getTotalSize());
        LOGGER.trace("getRawDataSize:" + writer.getRawDataSize());
        LOGGER.trace("getBitsetSize:" + writer.getBitsetSize());
        LOGGER.trace("getChunkOffsetHeaderSize:" + writer.getChunkOffsetHeaderSize());
        int dataLength = (int) (writer.getTotalSize() - writer.getChunkOffsetHeaderSize() - numBytesForBitmap);
        byte[] rawData = new byte[dataLength];
        // read the data segment that starts after the header.
        dis.read(rawData);
        Assert.assertEquals(rawData.length, byteArray.length);
        Assert.assertEquals(rawData, byteArray);
        raf.close();
        dis.close();
        file.delete();
        LOGGER.debug("END test maxBit:" + maxBits);
        maxBits = maxBits + 1;
        bitSet.close();
        customBit.close();
    }
}