Examples with FixedByteSingleValueMultiColWriter com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter used on opensource projects

Example 6 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColInt.

@Test
public void testSingleColInt() throws Exception {
    File file = new File("test_single_col_writer.dat");
    file.delete();
    int rows = 100;
    int cols = 1;
    int[] columnSizes = new int[] { 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, rows, cols, columnSizes);
    int[] data = new int[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextInt();
        writer.setInt(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getInt(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}

Example 7 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class SortedForwardIndexReaderTest method testSimple.

public void testSimple() throws Exception {
    int maxLength = 1000;
    int cardinality = 100000;
    File file = new File("test_sortef_fwd_index.dat");
    file.delete();
    int[] columnSizes = new int[] { 4, 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, cardinality, columnSizes.length, columnSizes);
    Random random = new Random();
    int[] startDocIdArray = new int[cardinality];
    int[] endDocIdArray = new int[cardinality];
    int prevEnd = -1;
    int totalDocs = 0;
    for (int i = 0; i < cardinality; i++) {
        int length = random.nextInt(maxLength);
        int start = prevEnd + 1;
        int end = start + length;
        startDocIdArray[i] = start;
        endDocIdArray[i] = end;
        writer.setInt(i, 0, start);
        writer.setInt(i, 1, end);
        prevEnd = end;
        totalDocs += length;
    }
    writer.close();
    PinotDataBuffer heapBuffer = PinotDataBuffer.fromFile(file, ReadMode.heap, FileChannel.MapMode.READ_ONLY, "testing");
    FixedByteSingleValueMultiColReader rawFileReader = new FixedByteSingleValueMultiColReader(heapBuffer, cardinality, columnSizes.length, columnSizes);
    SortedForwardIndexReader reader = new SortedForwardIndexReader(rawFileReader, totalDocs);
    // without using context
    long start, end;
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId), i);
        }
    }
    end = System.currentTimeMillis();
    System.out.println("Took " + (end - start) + " to scan " + totalDocs + " docs without using context");
    // with context
    SortedValueReaderContext context = reader.createContext();
    start = System.currentTimeMillis();
    for (int i = 0; i < cardinality; i++) {
        for (int docId = startDocIdArray[i]; docId <= endDocIdArray[i]; docId++) {
            Assert.assertEquals(reader.getInt(docId, context), i);
        }
    }
    end = System.currentTimeMillis();
    LOGGER.debug("Took " + (end - start) + " to scan " + totalDocs + " with context");
    reader.close();
    file.delete();
    heapBuffer.close();
}

Example 8 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColFloat.

@Test
public void testSingleColFloat() throws Exception {
    File wfile = new File("test_single_col_writer.dat");
    wfile.delete();
    final int rows = 100;
    final int cols = 1;
    final int[] columnSizes = new int[] { 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(wfile, rows, cols, columnSizes);
    final float[] data = new float[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextFloat();
        writer.setFloat(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getFloat(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}

Example 9 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testMultiCol.

@Test
public void testMultiCol() throws Exception {
    File file = new File("test_single_col_writer.dat");
    file.delete();
    int rows = 100;
    int cols = 2;
    int[] columnSizes = new int[] { 4, 4 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, rows, cols, columnSizes);
    int[][] data = new int[rows][cols];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        for (int j = 0; j < cols; j++) {
            data[i][j] = r.nextInt();
            writer.setInt(i, j, data[i][j]);
        }
    }
    writer.close();
    DataInputStream dis = new DataInputStream(new FileInputStream(file));
    for (int i = 0; i < rows; i++) {
        for (int j = 0; j < cols; j++) {
            Assert.assertEquals(dis.readInt(), data[i][j]);
        }
    }
    dis.close();
    file.delete();
}

Example 10 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColDouble.

@Test
public void testSingleColDouble() throws Exception {
    File wfile = new File("test_single_col_writer.dat");
    wfile.delete();
    final int rows = 100;
    final int cols = 1;
    final int[] columnSizes = new int[] { 8 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(wfile, rows, cols, columnSizes);
    final double[] data = new double[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextDouble();
        writer.setDouble(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getDouble(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}