Examples with FixedBitSingleValueMultiColReader com.linkedin.pinot.core.io.reader.impl.FixedBitSingleValueMultiColReader used on opensource projects

Example 1 with FixedBitSingleValueMultiColReader

use of com.linkedin.pinot.core.io.reader.impl.FixedBitSingleValueMultiColReader in project pinot by linkedin.

the class FixedBitWidthRowColDataFileReaderTest method testSingleColUnsigned.

/**
   * Tests only positive numbers
   *
   * @throws Exception
   */
@Test
public void testSingleColUnsigned() throws Exception {
    int[] maxBitArray = new int[] { 4 };
    for (int maxBits : maxBitArray) {
        String fileName = "test" + maxBits + "FixedBitWidthSingleCol";
        File file = new File(fileName);
        CustomBitSet bitset = null;
        try {
            //        System.out.println("START MAX BITS:" + maxBits);
            int numElements = 100;
            bitset = CustomBitSet.withBitLength(numElements * maxBits);
            int max = (int) Math.pow(2, maxBits);
            Random r = new Random();
            int[] values = new int[numElements];
            for (int i = 0; i < numElements; i++) {
                int value = r.nextInt(max);
                values[i] = value;
                for (int j = maxBits - 1; j >= 0; j--) {
                    if ((value & (1 << j)) != 0) {
                        bitset.setBit(i * maxBits + (maxBits - j - 1));
                    }
                }
            }
            byte[] byteArray = bitset.toByteArray();
            FileOutputStream fos = new FileOutputStream(file);
            fos.write(byteArray);
            fos.close();
            PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
            FixedBitSingleValueMultiColReader heapReader = new FixedBitSingleValueMultiColReader(heapBuffer, numElements, 1, new int[] { maxBits });
            for (int i = 0; i < numElements; i++) {
                int readInt = heapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            heapReader.close();
            heapBuffer.close();
            PinotDataBuffer dataBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap-testing");
            FixedBitSingleValueMultiColReader mmapReader = new FixedBitSingleValueMultiColReader(dataBuffer, numElements, 1, new int[] { maxBits });
            for (int i = 0; i < numElements; i++) {
                int readInt = mmapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            mmapReader.close();
            dataBuffer.close();
        //        System.out.println("END MAX BITS:" + maxBits);
        } finally {
            file.delete();
            bitset.close();
        }
    }
}

Example 2 with FixedBitSingleValueMultiColReader

use of com.linkedin.pinot.core.io.reader.impl.FixedBitSingleValueMultiColReader in project pinot by linkedin.

the class FixedBitWidthRowColDataFileReaderTest method testSingleColSigned.

/**
   * Tests both positive and negative numbers
   *
   * @throws Exception
   */
@Test
public void testSingleColSigned() throws Exception {
    int[] maxBitArray = new int[] { 4 };
    for (int maxBits : maxBitArray) {
        String fileName = "test" + maxBits + "FixedBitWidthSingleCol";
        File file = new File(fileName);
        CustomBitSet bitset = null;
        try {
            //        System.out.println("START MAX BITS:" + maxBits);
            int numElements = 100;
            int requiredBits = maxBits + 1;
            bitset = CustomBitSet.withBitLength(numElements * requiredBits);
            int max = (int) Math.pow(2, maxBits);
            Random r = new Random();
            int[] values = new int[numElements];
            int offset = max - 1;
            for (int i = 0; i < numElements; i++) {
                int value = r.nextInt(max);
                if (Math.random() > .5) {
                    value = -1 * value;
                }
                values[i] = value;
                int offsetValue = offset + value;
                for (int j = requiredBits - 1; j >= 0; j--) {
                    if ((offsetValue & (1 << j)) != 0) {
                        bitset.setBit(i * requiredBits + (requiredBits - j - 1));
                    }
                }
            }
            byte[] byteArray = bitset.toByteArray();
            FileOutputStream fos = new FileOutputStream(file);
            fos.write(byteArray);
            fos.close();
            PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
            FixedBitSingleValueMultiColReader heapReader = new FixedBitSingleValueMultiColReader(heapBuffer, numElements, 1, new int[] { maxBits }, new boolean[] { true });
            for (int i = 0; i < numElements; i++) {
                int readInt = heapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            heapReader.close();
            heapBuffer.close();
            PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap-testing");
            FixedBitSingleValueMultiColReader mmapReader = new FixedBitSingleValueMultiColReader(mmapBuffer, numElements, 1, new int[] { maxBits }, new boolean[] { true });
            for (int i = 0; i < numElements; i++) {
                int readInt = mmapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            mmapReader.close();
            mmapBuffer.close();
        //        System.out.println("END MAX BITS:" + maxBits);
        } finally {
            file.delete();
            bitset.close();
        }
    }
}

Example 3 with FixedBitSingleValueMultiColReader

use of com.linkedin.pinot.core.io.reader.impl.FixedBitSingleValueMultiColReader in project pinot by linkedin.

the class FixedBitRowColDataWriterReaderTest method testSingleColUnsigned.

@Test
public void testSingleColUnsigned() throws Exception {
    int maxBits = 1;
    while (maxBits < 32) {
        LOGGER.debug("START test maxBits:" + maxBits);
        final String fileName = getClass().getName() + "_single_col_fixed_bit_" + maxBits + ".dat";
        final File file = new File(fileName);
        file.delete();
        final int rows = 100;
        final int cols = 1;
        final int[] columnSizesInBits = new int[] { maxBits };
        final FixedBitSingleValueMultiColWriter writer = new FixedBitSingleValueMultiColWriter(file, rows, cols, columnSizesInBits, new boolean[] { true });
        final int[] data = new int[rows];
        final Random r = new Random();
        writer.open();
        final int maxValue = (int) Math.pow(2, maxBits);
        for (int i = 0; i < rows; i++) {
            data[i] = r.nextInt(maxValue) * ((Math.random() > .5) ? 1 : -1);
            writer.setInt(i, 0, data[i]);
        }
        writer.close();
        // Test heap mode
        PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
        FixedBitSingleValueMultiColReader heapReader = new FixedBitSingleValueMultiColReader(heapBuffer, rows, cols, columnSizesInBits, new boolean[] { true });
        for (int i = 0; i < rows; i++) {
            Assert.assertEquals(heapReader.getInt(i, 0), data[i]);
        }
        heapReader.close();
        heapBuffer.close();
        // Test mmap mode
        PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap-testing");
        FixedBitSingleValueMultiColReader mmapReader = new FixedBitSingleValueMultiColReader(mmapBuffer, rows, cols, columnSizesInBits, new boolean[] { true });
        for (int i = 0; i < rows; i++) {
            Assert.assertEquals(mmapReader.getInt(i, 0), data[i]);
        }
        mmapReader.close();
        mmapBuffer.close();
        maxBits = maxBits + 1;
        file.delete();
    }
}