Examples with FixedByteSingleValueMultiColReader com.linkedin.pinot.core.io.reader.impl.FixedByteSingleValueMultiColReader used on opensource projects

Example 11 with FixedByteSingleValueMultiColReader

use of com.linkedin.pinot.core.io.reader.impl.FixedByteSingleValueMultiColReader in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColLong.

@Test
public void testSingleColLong() throws Exception {
    File wfile = new File("test_single_col_writer.dat");
    wfile.delete();
    final int rows = 100;
    final int cols = 1;
    final int[] columnSizes = new int[] { 8 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(wfile, rows, cols, columnSizes);
    final long[] data = new long[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextLong();
        writer.setLong(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getLong(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}

Example 12 with FixedByteSingleValueMultiColReader

use of com.linkedin.pinot.core.io.reader.impl.FixedByteSingleValueMultiColReader in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColInt.

@Test
public void testSingleColInt() throws Exception {
    File file = new File("test_single_col_writer.dat");
    file.delete();
    int rows = 100;
    int cols = 1;
    int[] columnSizes = new int[] { 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, rows, cols, columnSizes);
    int[] data = new int[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextInt();
        writer.setInt(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getInt(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}

Example 13 with FixedByteSingleValueMultiColReader

use of com.linkedin.pinot.core.io.reader.impl.FixedByteSingleValueMultiColReader in project pinot by linkedin.

the class FixedByteWidthRowColDataFileReaderTest method testMultiCol.

@Test
void testMultiCol() throws Exception {
    String fileName = "test_single_col.dat";
    File f = new File(fileName);
    f.delete();
    DataOutputStream dos = new DataOutputStream(new FileOutputStream(f));
    int numRows = 100;
    int numCols = 2;
    int[][] colData = new int[numRows][numCols];
    Random r = new Random();
    for (int i = 0; i < numRows; i++) {
        colData[i] = new int[numCols];
        for (int j = 0; j < numCols; j++) {
            colData[i][j] = r.nextInt();
            dos.writeInt(colData[i][j]);
        }
    }
    dos.flush();
    dos.close();
    RandomAccessFile raf = new RandomAccessFile(f, "rw");
    //    System.out.println("file size: " + raf.getChannel().size());
    raf.close();
    PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(f, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing-heap");
    FixedByteSingleValueMultiColReader heapReader = new FixedByteSingleValueMultiColReader(heapBuffer, numRows, numCols, new int[] { 4, 4 });
    heapReader.open();
    for (int i = 0; i < numRows; i++) {
        for (int j = 0; j < numCols; j++) {
            Assert.assertEquals(heapReader.getInt(i, j), colData[i][j]);
        }
    }
    heapReader.close();
    PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(f, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "mmap_testing");
    FixedByteSingleValueMultiColReader mmapReader = new FixedByteSingleValueMultiColReader(mmapBuffer, numRows, numCols, new int[] { 4, 4 });
    mmapReader.open();
    for (int i = 0; i < numRows; i++) {
        for (int j = 0; j < numCols; j++) {
            Assert.assertEquals(mmapReader.getInt(i, j), colData[i][j]);
        }
    }
    mmapReader.close();
    f.delete();
}

Example 14 with FixedByteSingleValueMultiColReader

use of com.linkedin.pinot.core.io.reader.impl.FixedByteSingleValueMultiColReader in project pinot by linkedin.

the class SortedForwardIndexReaderTest method testSimple.

public void testSimple() throws Exception {
    int maxLength = 1000;
    int cardinality = 100000;
    File file = new File("test_sortef_fwd_index.dat");
    file.delete();
    int[] columnSizes = new int[] { 4, 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, cardinality, columnSizes.length, columnSizes);
    Random random = new Random();
    int[] startDocIdArray = new int[cardinality];
    int[] endDocIdArray = new int[cardinality];
    int prevEnd = -1;
    int totalDocs = 0;
    for (int i = 0; i < cardinality; i++) {
        int length = random.nextInt(maxLength);
        int start = prevEnd + 1;
        int end = start + length;
        startDocIdArray[i] = start;
        endDocIdArray[i] = end;
        writer.setInt(i, 0, start);
        writer.setInt(i, 1, end);
        prevEnd = end;
        totalDocs += length;
    }
    writer.close();
    PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
    FixedByteSingleValueMultiColReader rawFileReader = new FixedByteSingleValueMultiColReader(heapBuffer, cardinality, columnSizes.length, columnSizes);
    SortedForwardIndexReader reader = new SortedForwardIndexReader(rawFileReader, totalDocs);
    // without using context
    long start, end;
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId), i);
        }
    }
    end = System.currentTimeMillis();
    System.out.println("Took " + (end - start) + " to scan " + totalDocs + " docs without using context");
    // with context
    SortedValueReaderContext context = reader.createContext();
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId, context), i);
        }
    }
    end = System.currentTimeMillis();
    LOGGER.debug("Took " + (end - start) + " to scan " + totalDocs + " with context");
    reader.close();
    file.delete();
    heapBuffer.close();
}

Example 15 with FixedByteSingleValueMultiColReader

use of com.linkedin.pinot.core.io.reader.impl.FixedByteSingleValueMultiColReader in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColFloat.

@Test
public void testSingleColFloat() throws Exception {
    File wfile = new File("test_single_col_writer.dat");
    wfile.delete();
    final int rows = 100;
    final int cols = 1;
    final int[] columnSizes = new int[] { 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(wfile, rows, cols, columnSizes);
    final float[] data = new float[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextFloat();
        writer.setFloat(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getFloat(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}