Examples with PinotDataBuffer com.linkedin.pinot.core.segment.memory.PinotDataBuffer used on opensource projects

Example 41 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedByteChunkSingleValueReaderWriteTest method testFloat.

@Test
public void testFloat() throws Exception {
    float[] expected = new float[NUM_VALUES];
    for (int i = 0; i < NUM_VALUES; i++) {
        expected[i] = _random.nextFloat();
    }
    File outFile = new File(TEST_FILE);
    FileUtils.deleteQuietly(outFile);
    ChunkCompressor compressor = ChunkCompressorFactory.getCompressor("snappy");
    FixedByteChunkSingleValueWriter writer = new FixedByteChunkSingleValueWriter(outFile, compressor, NUM_VALUES, NUM_DOCS_PER_CHUNK, V1Constants.Numbers.FLOAT_SIZE);
    for (int i = 0; i < NUM_VALUES; i++) {
        writer.setFloat(i, expected[i]);
    }
    writer.close();
    PinotDataBuffer pinotDataBuffer = PinotDataBuffer.fromFile(outFile, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, getClass().getName());
    ChunkDecompressor uncompressor = ChunkCompressorFactory.getDecompressor("snappy");
    FixedByteChunkSingleValueReader reader = new FixedByteChunkSingleValueReader(pinotDataBuffer, uncompressor);
    ChunkReaderContext context = reader.createContext();
    for (int i = 0; i < NUM_VALUES; i++) {
        float actual = reader.getFloat(i, context);
        Assert.assertEquals(actual, expected[i]);
    }
    reader.close();
    FileUtils.deleteQuietly(outFile);
}

Example 42 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedByteChunkSingleValueReaderWriteTest method testDouble.

@Test
public void testDouble() throws Exception {
    double[] expected = new double[NUM_VALUES];
    for (int i = 0; i < NUM_VALUES; i++) {
        expected[i] = _random.nextDouble();
    }
    File outFile = new File(TEST_FILE);
    FileUtils.deleteQuietly(outFile);
    ChunkCompressor compressor = ChunkCompressorFactory.getCompressor("snappy");
    FixedByteChunkSingleValueWriter writer = new FixedByteChunkSingleValueWriter(outFile, compressor, NUM_VALUES, NUM_DOCS_PER_CHUNK, V1Constants.Numbers.DOUBLE_SIZE);
    for (int i = 0; i < NUM_VALUES; i++) {
        writer.setDouble(i, expected[i]);
    }
    writer.close();
    PinotDataBuffer pinotDataBuffer = PinotDataBuffer.fromFile(outFile, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, getClass().getName());
    ChunkDecompressor uncompressor = ChunkCompressorFactory.getDecompressor("snappy");
    FixedByteChunkSingleValueReader reader = new FixedByteChunkSingleValueReader(pinotDataBuffer, uncompressor);
    ChunkReaderContext context = reader.createContext();
    for (int i = 0; i < NUM_VALUES; i++) {
        double actual = reader.getDouble(i, context);
        Assert.assertEquals(actual, expected[i]);
    }
    reader.close();
    FileUtils.deleteQuietly(outFile);
}

Example 43 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColFloat.

@Test
public void testSingleColFloat() throws Exception {
    File wfile = new File("test_single_col_writer.dat");
    wfile.delete();
    final int rows = 100;
    final int cols = 1;
    final int[] columnSizes = new int[] { 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(wfile, rows, cols, columnSizes);
    final float[] data = new float[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextFloat();
        writer.setFloat(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getFloat(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}

Example 44 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColDouble.

@Test
public void testSingleColDouble() throws Exception {
    File wfile = new File("test_single_col_writer.dat");
    wfile.delete();
    final int rows = 100;
    final int cols = 1;
    final int[] columnSizes = new int[] { 8 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(wfile, rows, cols, columnSizes);
    final double[] data = new double[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextDouble();
        writer.setDouble(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getDouble(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}

Example 45 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedBitMultiValueTest method testSingleColMultiValueWithContext.

public void testSingleColMultiValueWithContext(Class<? extends SingleColumnMultiValueWriter> writerClazz, Class<? extends SingleColumnMultiValueReader<? extends ReaderContext>> readerClazz) throws Exception {
    LOGGER.info("Testing for writerClazz:{} readerClass:{}", writerClazz.getName(), readerClazz.getName());
    Constructor<? extends SingleColumnMultiValueWriter> writerClazzConstructor = writerClazz.getConstructor(new Class[] { File.class, int.class, int.class, int.class });
    Constructor<? extends SingleColumnMultiValueReader<? extends ReaderContext>> readerClazzConstructor = readerClazz.getConstructor(new Class[] { PinotDataBuffer.class, int.class, int.class, int.class, boolean.class });
    int maxBits = 1;
    while (maxBits < 32) {
        final String fileName = getClass().getName() + "_test_single_col_mv_fixed_bit.dat";
        final File f = new File(fileName);
        f.delete();
        int numDocs = 10;
        int maxNumValues = 100;
        final int[][] data = new int[numDocs][];
        final Random r = new Random();
        final int maxValue = (int) Math.pow(2, maxBits);
        int totalNumValues = 0;
        int[] startOffsets = new int[numDocs];
        int[] lengths = new int[numDocs];
        for (int i = 0; i < data.length; i++) {
            final int numValues = r.nextInt(maxNumValues) + 1;
            data[i] = new int[numValues];
            for (int j = 0; j < numValues; j++) {
                data[i][j] = r.nextInt(maxValue);
            }
            startOffsets[i] = totalNumValues;
            lengths[i] = numValues;
            totalNumValues = totalNumValues + numValues;
        }
        SingleColumnMultiValueWriter writer = writerClazzConstructor.newInstance(new Object[] { f, numDocs, totalNumValues, maxBits });
        for (int i = 0; i < data.length; i++) {
            writer.setIntArray(i, data[i]);
        }
        writer.close();
        final RandomAccessFile raf = new RandomAccessFile(f, "rw");
        raf.close();
        // Test heap mode
        PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(f, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
        SingleColumnMultiValueReader heapReader = readerClazzConstructor.newInstance(new Object[] { heapBuffer, numDocs, totalNumValues, maxBits, false });
        ReaderContext context = heapReader.createContext();
        final int[] readValues = new int[maxNumValues];
        for (int i = 0; i < data.length; i++) {
            final int numValues = heapReader.getIntArray(i, readValues, context);
            if (numValues != data[i].length) {
                System.err.println("Failed Expected:" + data[i].length + " Actual:" + numValues);
                int length = heapReader.getIntArray(i, readValues, context);
            }
            Assert.assertEquals(numValues, data[i].length);
            for (int j = 0; j < numValues; j++) {
                Assert.assertEquals(readValues[j], data[i][j]);
            }
        }
        heapReader.close();
        heapBuffer.close();
        // Test mmap mode
        PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(f, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
        SingleColumnMultiValueReader<? extends ReaderContext> mmapReader = readerClazzConstructor.newInstance(new Object[] { mmapBuffer, numDocs, totalNumValues, maxBits, false });
        for (int i = 0; i < data.length; i++) {
            final int numValues = mmapReader.getIntArray(i, readValues);
            Assert.assertEquals(numValues, data[i].length);
            for (int j = 0; j < numValues; j++) {
                Assert.assertEquals(readValues[j], data[i][j]);
            }
        }
        mmapReader.close();
        mmapBuffer.close();
        f.delete();
        maxBits = maxBits + 1;
    }
    LOGGER.info("DONE: Testing for writerClazz:{} readerClass:{}", writerClazz.getName(), readerClazz.getName());
}