Examples with MeasureRawColumnChunk org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk used on opensource projects

Example 16 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV1 method readRawMeasureChunk.

/**
 * Method to read the blocks data based on block index
 *
 * @param fileReader file reader to read the blocks
 * @param columnIndex column to be read
 * @return measure data chunk
 */
@Override
public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException {
    DataChunk dataChunk = measureColumnChunks.get(columnIndex);
    ByteBuffer buffer = fileReader.readByteBuffer(filePath, dataChunk.getDataPageOffset(), dataChunk.getDataPageLength());
    MeasureRawColumnChunk rawColumnChunk = new MeasureRawColumnChunk(columnIndex, buffer, 0, dataChunk.getDataPageLength(), this);
    rawColumnChunk.setFileReader(fileReader);
    rawColumnChunk.setPagesCount(1);
    rawColumnChunk.setRowCount(new int[] { numberOfRows });
    return rawColumnChunk;
}

Example 17 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV2 method readRawMeasureChunksInGroup.

/**
 * Below method will be used to read measure chunk data in group.
 * This method will be useful to avoid multiple IO while reading the
 * data from
 *
 * @param fileReader               file reader to read the data
 * @param startColumnIndex first column blocklet index to be read
 * @param endColumnIndex   end column blocklet index to be read
 * @return measure raw chunkArray
 * @throws IOException
 */
protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, int startColumnIndex, int endColumnIndex) throws IOException {
    long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex);
    ByteBuffer buffer = null;
    synchronized (fileReader) {
        buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset, (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset));
    }
    MeasureRawColumnChunk[] dataChunks = new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1];
    int runningLength = 0;
    int index = 0;
    for (int i = startColumnIndex; i <= endColumnIndex; i++) {
        int currentLength = (int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
        MeasureRawColumnChunk measureRawColumnChunk = new MeasureRawColumnChunk(i, buffer, runningLength, currentLength, this);
        measureRawColumnChunk.setRowCount(new int[] { numberOfRows });
        measureRawColumnChunk.setFileReader(fileReader);
        measureRawColumnChunk.setPagesCount(1);
        dataChunks[index] = measureRawColumnChunk;
        runningLength += currentLength;
        index++;
    }
    return dataChunks;
}

Example 18 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV2 method readRawMeasureChunk.

@Override
public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex) throws IOException {
    int dataLength = 0;
    if (measureColumnChunkOffsets.size() - 1 == columnIndex) {
        DataChunk2 metadataChunk = null;
        synchronized (fileReader) {
            metadataChunk = CarbonUtil.readDataChunk(ByteBuffer.wrap(fileReader.readByteArray(filePath, measureColumnChunkOffsets.get(columnIndex), measureColumnChunkLength.get(columnIndex))), 0, measureColumnChunkLength.get(columnIndex));
        }
        dataLength = measureColumnChunkLength.get(columnIndex) + metadataChunk.data_page_length;
    } else {
        long currentMeasureOffset = measureColumnChunkOffsets.get(columnIndex);
        dataLength = (int) (measureColumnChunkOffsets.get(columnIndex + 1) - currentMeasureOffset);
    }
    ByteBuffer buffer = null;
    synchronized (fileReader) {
        buffer = fileReader.readByteBuffer(filePath, measureColumnChunkOffsets.get(columnIndex), dataLength);
    }
    MeasureRawColumnChunk rawColumnChunk = new MeasureRawColumnChunk(columnIndex, buffer, 0, dataLength, this);
    rawColumnChunk.setFileReader(fileReader);
    rawColumnChunk.setPagesCount(1);
    rawColumnChunk.setRowCount(new int[] { numberOfRows });
    return rawColumnChunk;
}

Example 19 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV3 method readRawMeasureChunksInGroup.

/**
 * Below method will be used to read the multiple measure column data in group
 * and divide into measure raw chunk object
 * Steps for reading
 * 1. Get the length of the data to be read
 * 2. Allocate the direct buffer
 * 3. read the data from file
 * 4. Get the data chunk object from file for each column
 * 5. Create the raw chunk object and fill the details for each column
 * 6. increment the offset of the data
 *
 * @param fileReader
 *        reader which will be used to read the measure columns data from file
 * @param startColumnIndex
 *        column index of the first measure column
 * @param endColumnIndex
 *        column index of the last measure column
 * @return MeasureRawColumnChunk array
 */
protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader, int startColumnIndex, int endColumnIndex) throws IOException {
    // to calculate the length of the data to be read
    // column we can subtract the offset of start column offset with
    // end column+1 offset and get the total length.
    long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex);
    ByteBuffer buffer = null;
    // read the data from carbon data file
    synchronized (fileReader) {
        buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset, (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset));
    }
    // create raw chunk for each measure column
    MeasureRawColumnChunk[] measureDataChunk = new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1];
    int runningLength = 0;
    int index = 0;
    for (int i = startColumnIndex; i <= endColumnIndex; i++) {
        int currentLength = (int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
        DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, runningLength, measureColumnChunkLength.get(i));
        MeasureRawColumnChunk measureRawColumnChunk = getMeasureRawColumnChunk(fileReader, i, runningLength, currentLength, buffer, dataChunk);
        measureDataChunk[index] = measureRawColumnChunk;
        runningLength += currentLength;
        index++;
    }
    return measureDataChunk;
}

Example 20 with MeasureRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV3 method getMeasureRawColumnChunk.

MeasureRawColumnChunk getMeasureRawColumnChunk(FileReader fileReader, int columnIndex, long offset, int dataLength, ByteBuffer buffer, DataChunk3 dataChunk) {
    // creating a raw chunks instance and filling all the details
    MeasureRawColumnChunk rawColumnChunk = new MeasureRawColumnChunk(columnIndex, buffer, offset, dataLength, this);
    int numberOfPages = dataChunk.getPage_length().size();
    byte[][] maxValueOfEachPage = new byte[numberOfPages][];
    byte[][] minValueOfEachPage = new byte[numberOfPages][];
    int[] eachPageLength = new int[numberOfPages];
    for (int i = 0; i < minValueOfEachPage.length; i++) {
        maxValueOfEachPage[i] = dataChunk.getData_chunk_list().get(i).getMin_max().getMax_values().get(0).array();
        minValueOfEachPage[i] = dataChunk.getData_chunk_list().get(i).getMin_max().getMin_values().get(0).array();
        eachPageLength[i] = dataChunk.getData_chunk_list().get(i).getNumberOfRowsInpage();
    }
    rawColumnChunk.setDataChunkV3(dataChunk);
    rawColumnChunk.setFileReader(fileReader);
    rawColumnChunk.setPagesCount(dataChunk.getPage_length().size());
    rawColumnChunk.setMaxValues(maxValueOfEachPage);
    rawColumnChunk.setMinValues(minValueOfEachPage);
    rawColumnChunk.setRowCount(eachPageLength);
    rawColumnChunk.setOffsets(ArrayUtils.toPrimitive(dataChunk.page_offset.toArray(new Integer[dataChunk.page_offset.size()])));
    return rawColumnChunk;
}