Examples with FixedByteSingleValueMultiColWriter com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter used on opensource projects

Example 1 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteSingleColumnMultiValueReaderWriter method addCapacity.

/**
   * This method automatically computes the space needed based on the columnSizeInBytes
   * @param rowCapacity Additional capacity to be added in terms of number of rows
   * @throws RuntimeException
   */
private void addCapacity(int rowCapacity) throws RuntimeException {
    PinotDataBuffer dataBuffer;
    try {
        dataBuffer = PinotDataBuffer.allocateDirect(rowCapacity * columnSizeInBytes);
        //dataBuffer.order(ByteOrder.nativeOrder());
        dataBuffers.add(dataBuffer);
        currentDataWriter = new FixedByteSingleValueMultiColWriter(dataBuffer, rowCapacity, 1, new int[] { columnSizeInBytes });
        dataWriters.add(currentDataWriter);
        FixedByteSingleValueMultiColReader dataFileReader = new FixedByteSingleValueMultiColReader(dataBuffer, rowCapacity, 1, new int[] { columnSizeInBytes });
        dataReaders.add(dataFileReader);
        //update the capacity
        currentCapacity = rowCapacity;
        currentDataWriterIndex = currentDataWriterIndex + 1;
    } catch (Exception e) {
        throw new RuntimeException("Error while expanding the capacity by allocating additional buffer with capacity:" + rowCapacity, e);
    }
}

Example 2 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteSingleColumnMultiValueReaderWriter method init.

private void init(int rows, int columnSizeInBytes, int maxNumberOfMultiValuesPerRow, int initialCapacity, int incrementalCapacity) throws IOException {
    this.columnSizeInBytes = columnSizeInBytes;
    this.maxNumberOfMultiValuesPerRow = maxNumberOfMultiValuesPerRow;
    headerSize = rows * SIZE_OF_INT * NUM_COLS_IN_HEADER;
    headerBuffer = PinotDataBuffer.allocateDirect(headerSize);
    //    headerBuffer.order(ByteOrder.nativeOrder());
    //dataBufferId, startIndex, length
    headerWriter = new FixedByteSingleValueMultiColWriter(headerBuffer, rows, 3, new int[] { SIZE_OF_INT, SIZE_OF_INT, SIZE_OF_INT });
    headerReader = new FixedByteSingleValueMultiColReader(headerBuffer, rows, 3, new int[] { SIZE_OF_INT, SIZE_OF_INT, SIZE_OF_INT });
    //at least create space for million entries, which for INT translates into 4mb buffer
    this.incrementalCapacity = incrementalCapacity;
    addCapacity(initialCapacity);
}

Example 3 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSpecialPaddingCharsForStringReaderWriter.

@Test
public void testSpecialPaddingCharsForStringReaderWriter() throws Exception {
    for (int iter = 0; iter < 2; iter++) {
        char paddingChar = (iter == 0) ? '%' : '\0';
        final byte[] bytes1 = new byte[] { -17, -65, -67, -17, -65, -67, 32, 69, 120, 101, 99, 117, 116, 105, 118, 101 };
        final byte[] bytes2 = new byte[] { -17, -65, -68, 32, 99, 97, 108, 103, 97, 114, 121, 32, 106, 117, 110, 107, 32, 114, 101, 109, 111, 118, 97, 108 };
        File file = new File("test_single_col_writer.dat");
        file.delete();
        int rows = 100;
        int cols = 1;
        String testString1 = new String(bytes1);
        String testString2 = new String(bytes2);
        //      System.out.println(Arrays.toString(bytes2));
        int stringColumnMaxLength = Math.max(testString1.getBytes().length, testString2.getBytes().length);
        int[] columnSizes = new int[] { stringColumnMaxLength };
        FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, rows, cols, columnSizes);
        String[] data = new String[rows];
        for (int i = 0; i < rows; i++) {
            String toPut = (i % 2 == 0) ? testString1 : testString2;
            final int padding = stringColumnMaxLength - toPut.getBytes().length;
            final StringBuilder bld = new StringBuilder();
            bld.append(toPut);
            for (int j = 0; j < padding; j++) {
                bld.append(paddingChar);
            }
            data[i] = bld.toString();
            writer.setString(i, 0, data[i]);
        }
        writer.close();
        PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
        FixedByteSingleValueMultiColReader dataFileReader = new FixedByteSingleValueMultiColReader(mmapBuffer, rows, 1, new int[] { stringColumnMaxLength });
        for (int i = 0; i < rows; i++) {
            String stringInFile = dataFileReader.getString(i, 0);
            Assert.assertEquals(stringInFile, data[i]);
            Assert.assertEquals(StringUtils.remove(stringInFile, String.valueOf(paddingChar)), StringUtils.remove(data[i], String.valueOf(paddingChar)));
        }
        file.delete();
    }
}

Example 4 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSpecialCharsForStringReaderWriter.

@Test
public void testSpecialCharsForStringReaderWriter() throws Exception {
    final byte[] bytes1 = new byte[] { -17, -65, -67, -17, -65, -67, 32, 69, 120, 101, 99, 117, 116, 105, 118, 101 };
    final byte[] bytes2 = new byte[] { -17, -65, -68, 32, 99, 97, 108, 103, 97, 114, 121, 32, 106, 117, 110, 107, 32, 114, 101, 109, 111, 118, 97, 108 };
    File file = new File("test_single_col_writer.dat");
    file.delete();
    int rows = 100;
    int cols = 1;
    String testString1 = new String(bytes1);
    String testString2 = new String(bytes2);
    //    System.out.println(Arrays.toString(bytes2));
    int stringColumnMaxLength = Math.max(testString1.getBytes().length, testString2.getBytes().length);
    int[] columnSizes = new int[] { stringColumnMaxLength };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(file, rows, cols, columnSizes);
    String[] data = new String[rows];
    for (int i = 0; i < rows; i++) {
        String toPut = (i % 2 == 0) ? testString1 : testString2;
        final int padding = stringColumnMaxLength - toPut.getBytes().length;
        final StringBuilder bld = new StringBuilder();
        bld.append(toPut);
        for (int j = 0; j < padding; j++) {
            bld.append(V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
        }
        data[i] = bld.toString();
        writer.setString(i, 0, data[i]);
    }
    writer.close();
    PinotDataBuffer mmapBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing");
    FixedByteSingleValueMultiColReader dataFileReader = new FixedByteSingleValueMultiColReader(mmapBuffer, rows, 1, new int[] { stringColumnMaxLength });
    for (int i = 0; i < rows; i++) {
        String stringInFile = dataFileReader.getString(i, 0);
        Assert.assertEquals(stringInFile, data[i]);
        Assert.assertEquals(StringUtils.remove(stringInFile, String.valueOf(V1Constants.Str.DEFAULT_STRING_PAD_CHAR)), StringUtils.remove(data[i], String.valueOf(V1Constants.Str.DEFAULT_STRING_PAD_CHAR)));
    }
    file.delete();
}

Example 5 with FixedByteSingleValueMultiColWriter

use of com.linkedin.pinot.core.io.writer.impl.FixedByteSingleValueMultiColWriter in project pinot by linkedin.

the class FixedByteWidthRowColDataFileWriterTest method testSingleColLong.

@Test
public void testSingleColLong() throws Exception {
    File wfile = new File("test_single_col_writer.dat");
    wfile.delete();
    final int rows = 100;
    final int cols = 1;
    final int[] columnSizes = new int[] { 8 };
    FixedByteSingleValueMultiColWriter writer = new FixedByteSingleValueMultiColWriter(wfile, rows, cols, columnSizes);
    final long[] data = new long[rows];
    Random r = new Random();
    for (int i = 0; i < rows; i++) {
        data[i] = r.nextLong();
        writer.setLong(i, 0, data[i]);
    }
    writer.close();
    File rfile = new File("test_single_col_writer.dat");
    PinotDataBuffer buffer = PinotDataBuffer.fromFile(rfile, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, "testing");
    FixedByteSingleValueMultiColReader reader = new FixedByteSingleValueMultiColReader(buffer, rows, cols, columnSizes);
    for (int i = 0; i < rows; i++) {
        Assert.assertEquals(reader.getLong(i, 0), data[i]);
    }
    reader.close();
    rfile.delete();
}