Examples with PinotDataBuffer com.linkedin.pinot.core.segment.memory.PinotDataBuffer used on opensource projects

Example 36 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedBitWidthRowColDataFileReaderTest method testSingleColSigned.

/**
   * Tests both positive and negative numbers
   *
   * @throws Exception
   */
@Test
public void testSingleColSigned() throws Exception {
    int[] maxBitArray = new int[] { 4 };
    for (int maxBits : maxBitArray) {
        String fileName = "test" + maxBits + "FixedBitWidthSingleCol";
        File file = new File(fileName);
        CustomBitSet bitset = null;
        try {
            //        System.out.println("START MAX BITS:" + maxBits);
            int numElements = 100;
            int requiredBits = maxBits + 1;
            bitset = CustomBitSet.withBitLength(numElements * requiredBits);
            int max = (int) Math.pow(2, maxBits);
            Random r = new Random();
            int[] values = new int[numElements];
            int offset = max - 1;
            for (int i = 0; i < numElements; i++) {
                int value = r.nextInt(max);
                if (Math.random() > .5) {
                    value = -1 * value;
                }
                values[i] = value;
                int offsetValue = offset + value;
                for (int j = requiredBits - 1; j >= 0; j--) {
                    if ((offsetValue & (1 << j)) != 0) {
                        bitset.setBit(i * requiredBits + (requiredBits - j - 1));
                    }
                }
            }
            byte[] byteArray = bitset.toByteArray();
            FileOutputStream fos = new FileOutputStream(file);
            fos.write(byteArray);
            fos.close();
            PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
            FixedBitSingleValueMultiColReader heapReader = new FixedBitSingleValueMultiColReader(heapBuffer, numElements, 1, new int[] { maxBits }, new boolean[] { true });
            for (int i = 0; i < numElements; i++) {
                int readInt = heapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            heapReader.close();
            heapBuffer.close();
            PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap-testing");
            FixedBitSingleValueMultiColReader mmapReader = new FixedBitSingleValueMultiColReader(mmapBuffer, numElements, 1, new int[] { maxBits }, new boolean[] { true });
            for (int i = 0; i < numElements; i++) {
                int readInt = mmapReader.getInt(i, 0);
                Assert.assertEquals(readInt, values[i]);
            }
            mmapReader.close();
            mmapBuffer.close();
        //        System.out.println("END MAX BITS:" + maxBits);
        } finally {
            file.delete();
            bitset.close();
        }
    }
}

Example 37 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedByteSkipListSCMVReaderTest method testSingleColMultiValue.

@Test
public void testSingleColMultiValue() throws Exception {
    String fileName = "test_single_col.dat";
    File f = new File(fileName);
    f.delete();
    int numDocs = 100;
    int maxMultiValues = 100;
    int[][] data = new int[numDocs][];
    Random r = new Random();
    int totalNumValues = 0;
    int[] startOffsets = new int[numDocs];
    int[] lengths = new int[numDocs];
    for (int i = 0; i < data.length; i++) {
        int numValues = r.nextInt(maxMultiValues) + 1;
        data[i] = new int[numValues];
        for (int j = 0; j < numValues; j++) {
            data[i][j] = r.nextInt();
        }
        startOffsets[i] = totalNumValues;
        lengths[i] = numValues;
        totalNumValues = totalNumValues + numValues;
    }
    for (int i = 0; i < data.length; i++) {
        LOGGER.trace("(" + i + "," + startOffsets[i] + "," + lengths[i] + ")");
        LOGGER.trace(",");
    }
    LOGGER.trace(Arrays.toString(startOffsets));
    LOGGER.trace(Arrays.toString(lengths));
    FixedByteSkipListMultiValueWriter writer = new FixedByteSkipListMultiValueWriter(f, numDocs, totalNumValues, Integer.SIZE / 8);
    for (int i = 0; i < data.length; i++) {
        writer.setIntArray(i, data[i]);
    }
    writer.close();
    int[] readValues = new int[maxMultiValues];
    PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(f, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "heap-testing");
    FixedByteMultiValueReader heapReader = new FixedByteMultiValueReader(heapBuffer, numDocs, totalNumValues, Integer.SIZE / 8);
    for (int i = 0; i < data.length; i++) {
        int numValues = heapReader.getIntArray(i, readValues);
        Assert.assertEquals(numValues, data[i].length);
        for (int j = 0; j < numValues; j++) {
            Assert.assertEquals(readValues[j], data[i][j], "");
        }
    }
    heapReader.close();
    heapBuffer.close();
    PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(f, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap-testing");
    FixedByteMultiValueReader mmapReader = new FixedByteMultiValueReader(mmapBuffer, numDocs, totalNumValues, Integer.SIZE / 8);
    for (int i = 0; i < data.length; i++) {
        int numValues = mmapReader.getIntArray(i, readValues);
        Assert.assertEquals(numValues, data[i].length);
        for (int j = 0; j < numValues; j++) {
            Assert.assertEquals(readValues[j], data[i][j], "");
        }
    }
    mmapReader.close();
    mmapBuffer.close();
    f.delete();
}

Example 38 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedByteWidthRowColDataFileReaderTest method testMultiCol.

@Test
void testMultiCol() throws Exception {
    String fileName = "test_single_col.dat";
    File f = new File(fileName);
    f.delete();
    DataOutputStream dos = new DataOutputStream(new FileOutputStream(f));
    int numRows = 100;
    int numCols = 2;
    int[][] colData = new int[numRows][numCols];
    Random r = new Random();
    for (int i = 0; i < numRows; i++) {
        colData[i] = new int[numCols];
        for (int j = 0; j < numCols; j++) {
            colData[i][j] = r.nextInt();
            dos.writeInt(colData[i][j]);
        }
    }
    dos.flush();
    dos.close();
    RandomAccessFile raf = new RandomAccessFile(f, "rw");
    //    System.out.println("file size: " + raf.getChannel().size());
    raf.close();
    PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(f, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing-heap");
    FixedByteSingleValueMultiColReader heapReader = new FixedByteSingleValueMultiColReader(heapBuffer, numRows, numCols, new int[] { 4, 4 });
    heapReader.open();
    for (int i = 0; i < numRows; i++) {
        for (int j = 0; j < numCols; j++) {
            Assert.assertEquals(heapReader.getInt(i, j), colData[i][j]);
        }
    }
    heapReader.close();
    PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(f, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap_testing");
    FixedByteSingleValueMultiColReader mmapReader = new FixedByteSingleValueMultiColReader(mmapBuffer, numRows, numCols, new int[] { 4, 4 });
    mmapReader.open();
    for (int i = 0; i < numRows; i++) {
        for (int j = 0; j < numCols; j++) {
            Assert.assertEquals(mmapReader.getInt(i, j), colData[i][j]);
        }
    }
    mmapReader.close();
    f.delete();
}

Example 39 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class SortedForwardIndexReaderTest method testSimple.

public void testSimple() throws Exception {
    int maxLength = 1000;
    int cardinality = 100000;
    File file = new File("test_sortef_fwd_index.dat");
    file.delete();
    int[] columnSizes = new int[] { 4, 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, cardinality, columnSizes.length, columnSizes);
    Random random = new Random();
    int[] startDocIdArray = new int[cardinality];
    int[] endDocIdArray = new int[cardinality];
    int prevEnd = -1;
    int totalDocs = 0;
    for (int i = 0; i < cardinality; i++) {
        int length = random.nextInt(maxLength);
        int start = prevEnd + 1;
        int end = start + length;
        startDocIdArray[i] = start;
        endDocIdArray[i] = end;
        writer.setInt(i, 0, start);
        writer.setInt(i, 1, end);
        prevEnd = end;
        totalDocs += length;
    }
    writer.close();
    PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
    FixedByteSingleValueMultiColReader rawFileReader = new FixedByteSingleValueMultiColReader(heapBuffer, cardinality, columnSizes.length, columnSizes);
    SortedForwardIndexReader reader = new SortedForwardIndexReader(rawFileReader, totalDocs);
    // without using context
    long start, end;
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId), i);
        }
    }
    end = System.currentTimeMillis();
    System.out.println("Took " + (end - start) + " to scan " + totalDocs + " docs without using context");
    // with context
    SortedValueReaderContext context = reader.createContext();
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId, context), i);
        }
    }
    end = System.currentTimeMillis();
    LOGGER.debug("Took " + (end - start) + " to scan " + totalDocs + " with context");
    reader.close();
    file.delete();
    heapBuffer.close();
}

Example 40 with PinotDataBuffer

use of com.linkedin.pinot.core.segment.memory.PinotDataBuffer in project pinot by linkedin.

the class FixedByteChunkSingleValueReaderWriteTest method testLong.

@Test
public void testLong() throws Exception {
    long[] expected = new long[NUM_VALUES];
    for (int i = 0; i < NUM_VALUES; i++) {
        expected[i] = _random.nextLong();
    }
    File outFile = new File(TEST_FILE);
    FileUtils.deleteQuietly(outFile);
    ChunkCompressor compressor = ChunkCompressorFactory.getCompressor("snappy");
    FixedByteChunkSingleValueWriter writer = new FixedByteChunkSingleValueWriter(outFile, compressor, NUM_VALUES, NUM_DOCS_PER_CHUNK, V1Constants.Numbers.LONG_SIZE);
    for (int i = 0; i < NUM_VALUES; i++) {
        writer.setLong(i, expected[i]);
    }
    writer.close();
    PinotDataBuffer pinotDataBuffer = PinotDataBuffer.fromFile(outFile, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, getClass().getName());
    ChunkDecompressor uncompressor = ChunkCompressorFactory.getDecompressor("snappy");
    FixedByteChunkSingleValueReader reader = new FixedByteChunkSingleValueReader(pinotDataBuffer, uncompressor);
    ChunkReaderContext context = reader.createContext();
    for (int i = 0; i < NUM_VALUES; i++) {
        long actual = reader.getLong(i, context);
        Assert.assertEquals(actual, expected[i]);
    }
    reader.close();
    FileUtils.deleteQuietly(outFile);
}