Examples with CustomBitSet com.linkedin.pinot.core.util.CustomBitSet used on opensource projects

Example 6 with CustomBitSet

use of com.linkedin.pinot.core.util.CustomBitSet in project pinot by linkedin.

the class FixedBitWidthRowColDataFileWriterTest method testSingleColSigned.

@Test
public void testSingleColSigned() throws Exception {
    int maxBits = 1;
    while (maxBits < 32) {
        LOGGER.trace("START test maxBits:" + maxBits);
        final String fileName = getClass().getName() + "_single_col_fixed_bit_" + maxBits + ".dat";
        final File file = new File(fileName);
        file.delete();
        final int rows = 100;
        final int cols = 1;
        final int[] columnSizesInBits = new int[] { maxBits };
        final FixedBitSingleValueMultiColWriter writer = new FixedBitSingleValueMultiColWriter(file, rows, cols, columnSizesInBits, new boolean[] { true });
        final int[] data = new int[rows];
        final Random r = new Random();
        writer.open();
        final int maxValue = (int) Math.pow(2, maxBits);
        int offset = maxValue - 1;
        int requiredBits = maxBits + 1;
        final CustomBitSet set = CustomBitSet.withBitLength(rows * cols * requiredBits);
        for (int i = 0; i < rows; i++) {
            int value = r.nextInt(maxValue);
            if (Math.random() > .5) {
                value = -1 * value;
            }
            data[i] = value;
            writer.setInt(i, 0, data[i]);
            int offsetValue = value + offset;
            for (int bitPos = requiredBits - 1; bitPos >= 0; bitPos--) {
                if ((offsetValue & (1 << bitPos)) != 0) {
                    set.setBit(i * requiredBits + (requiredBits - bitPos - 1));
                }
            }
        }
        LOGGER.trace("bits expected:" + rows * cols * maxBits + " bytes expected:" + (rows * cols * maxBits + 7) / 8);
        writer.close();
        final RandomAccessFile raf = new RandomAccessFile(file, "r");
        LOGGER.trace("file size:" + raf.length());
        final byte[] b = new byte[(int) raf.length()];
        raf.read(b);
        final byte[] byteArray = set.toByteArray();
        if (byteArray.length != raf.length()) {
            final byte[] temp = set.toByteArray();
            System.err.println("byteArray length:" + temp.length + "   from file:" + b.length);
        }
        LOGGER.trace("byteArray length:" + byteArray.length + "   from file:" + b.length);
        Assert.assertEquals(byteArray.length, b.length);
        Assert.assertEquals(byteArray, b);
        raf.close();
        LOGGER.trace("END test maxBits:" + maxBits);
        maxBits = maxBits + 1;
        file.delete();
        set.close();
    }
}

Example 7 with CustomBitSet

use of com.linkedin.pinot.core.util.CustomBitSet in project pinot by linkedin.

the class FixedBitWidthRowColDataFileWriterTest method testSingleColUnsigned.

@Test
public void testSingleColUnsigned() throws Exception {
    int maxBits = 1;
    while (maxBits < 32) {
        LOGGER.debug("START test maxBits:" + maxBits);
        final String fileName = getClass().getName() + "_single_col_fixed_bit_" + maxBits + ".dat";
        final File file = new File(fileName);
        file.delete();
        final int rows = 100;
        final int cols = 1;
        final int[] columnSizesInBits = new int[] { maxBits };
        final FixedBitSingleValueMultiColWriter writer = new FixedBitSingleValueMultiColWriter(file, rows, cols, columnSizesInBits);
        final int[] data = new int[rows];
        final Random r = new Random();
        writer.open();
        final int maxValue = (int) Math.pow(2, maxBits);
        final CustomBitSet set = CustomBitSet.withBitLength(rows * cols * maxBits);
        for (int i = 0; i < rows; i++) {
            data[i] = r.nextInt(maxValue);
            writer.setInt(i, 0, data[i]);
            final int value = data[i];
            for (int bitPos = maxBits - 1; bitPos >= 0; bitPos--) {
                if ((value & (1 << bitPos)) != 0) {
                    set.setBit(i * maxBits + (maxBits - bitPos - 1));
                }
            }
        }
        LOGGER.debug("bits expected:" + rows * cols * maxBits + " bytes expected:" + (rows * cols * maxBits + 7) / 8);
        writer.close();
        final RandomAccessFile raf = new RandomAccessFile(file, "r");
        //      System.out.println("file size:" + raf.length());
        final byte[] b = new byte[(int) raf.length()];
        raf.read(b);
        final byte[] byteArray = set.toByteArray();
        if (byteArray.length != raf.length()) {
            final byte[] temp = set.toByteArray();
            System.err.println("byteArray length:" + temp.length + "   from file:" + b.length);
        }
        LOGGER.trace("byteArray length:" + byteArray.length + "   from file:" + b.length);
        Assert.assertEquals(byteArray.length, b.length);
        Assert.assertEquals(byteArray, b);
        raf.close();
        LOGGER.trace("END test maxBits:" + maxBits);
        maxBits = maxBits + 1;
        file.delete();
        set.close();
    }
}

Example 8 with CustomBitSet

use of com.linkedin.pinot.core.util.CustomBitSet in project pinot by linkedin.

the class FixedBitWidthSingleColumnMultiValueWriterTest method testSingleColMultiValue.

@Test
public void testSingleColMultiValue() throws Exception {
    int maxBits = 2;
    while (maxBits < 32) {
        LOGGER.trace("START test maxBit:" + maxBits);
        File file = new File("test_single_col_multi_value_writer.dat");
        file.delete();
        int rows = 100;
        int[][] data = new int[rows][];
        int maxValue = (int) Math.pow(2, maxBits);
        Random r = new Random();
        int totalNumValues = 0;
        for (int i = 0; i < rows; i++) {
            int numValues = r.nextInt(100) + 1;
            data[i] = new int[numValues];
            for (int j = 0; j < numValues; j++) {
                data[i][j] = r.nextInt(maxValue);
            }
            totalNumValues += numValues;
        }
        FixedBitSingleColumnMultiValueWriter writer = new FixedBitSingleColumnMultiValueWriter(file, rows, totalNumValues, maxBits);
        CustomBitSet bitSet = CustomBitSet.withBitLength(totalNumValues * maxBits);
        int index = 0;
        for (int i = 0; i < rows; i++) {
            writer.setIntArray(i, data[i]);
            for (int j = 0; j < data[i].length; j++) {
                int value = data[i][j];
                for (int bitPos = maxBits - 1; bitPos >= 0; bitPos--) {
                    if ((value & (1 << bitPos)) != 0) {
                        bitSet.setBit(index * maxBits + (maxBits - bitPos - 1));
                    }
                }
                index = index + 1;
            }
        }
        writer.close();
        // verify header
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        int totalLength = 0;
        for (int i = 0; i < rows; i++) {
            Assert.assertEquals(dis.readInt(), totalLength);
            Assert.assertEquals(dis.readInt(), data[i].length);
            totalLength += data[i].length;
        }
        dis.close();
        // verify data
        byte[] byteArray = bitSet.toByteArray();
        RandomAccessFile raf = new RandomAccessFile(file, "r");
        // Header contains 1 row for each doc and each row contains 2 ints
        int headerSize = rows * 2 * 4;
        int dataLength = (int) raf.length() - headerSize;
        byte[] b = new byte[dataLength];
        // read the data segment that starts after the header.
        raf.seek(headerSize);
        raf.read(b, 0, dataLength);
        Assert.assertEquals(byteArray.length, b.length);
        Assert.assertEquals(byteArray, b);
        raf.close();
        file.delete();
        LOGGER.trace("END test maxBit:" + maxBits);
        maxBits = maxBits + 1;
        bitSet.close();
    }
}