Examples with BlockletEncodedColumnPage org.apache.carbondata.core.datastore.blocklet.BlockletEncodedColumnPage used on opensource projects

Example 1 with BlockletEncodedColumnPage

use of org.apache.carbondata.core.datastore.blocklet.BlockletEncodedColumnPage in project carbondata by apache.

the class CarbonMetadataUtil method getMeasureDataChunk3.

/**
 * return DataChunk3 for the measure column (specified by `columnIndex`)
 * in `encodedTablePageList`
 */
public static DataChunk3 getMeasureDataChunk3(EncodedBlocklet encodedBlocklet, int columnIndex) {
    List<DataChunk2> dataChunksList = new ArrayList<>();
    BlockletEncodedColumnPage blockletEncodedColumnPage = encodedBlocklet.getEncodedMeasureColumnPages().get(columnIndex);
    for (EncodedColumnPage encodedColumnPage : blockletEncodedColumnPage.getEncodedColumnPageList()) {
        dataChunksList.add(encodedColumnPage.getPageMetadata());
    }
    return CarbonMetadataUtil.getDataChunk3(dataChunksList, null);
}

Example 2 with BlockletEncodedColumnPage

use of org.apache.carbondata.core.datastore.blocklet.BlockletEncodedColumnPage in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method writeBlockletToFile.

/**
 * Write one blocklet data into file
 * File format:
 * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
 * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
 * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
 * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
 */
private void writeBlockletToFile(byte[][] dataChunkBytes) throws IOException {
    long offset = currentOffsetInFile;
    // to maintain the offset of each data chunk in blocklet
    List<Long> currentDataChunksOffset = new ArrayList<>();
    // to maintain the length of each data chunk in blocklet
    List<Integer> currentDataChunksLength = new ArrayList<>();
    EncodedBlocklet encodedBlocklet = blockletDataHolder.getEncodedBlocklet();
    int numberOfDimension = encodedBlocklet.getNumberOfDimension();
    int numberOfMeasures = encodedBlocklet.getNumberOfMeasure();
    ByteBuffer buffer = null;
    long dimensionOffset = 0;
    long measureOffset = 0;
    for (int i = 0; i < numberOfDimension; i++) {
        currentDataChunksOffset.add(offset);
        currentDataChunksLength.add(dataChunkBytes[i].length);
        buffer = ByteBuffer.wrap(dataChunkBytes[i]);
        currentOffsetInFile += fileChannel.write(buffer);
        offset += dataChunkBytes[i].length;
        BlockletEncodedColumnPage blockletEncodedColumnPage = encodedBlocklet.getEncodedDimensionColumnPages().get(i);
        for (EncodedColumnPage dimensionPage : blockletEncodedColumnPage.getEncodedColumnPageList()) {
            buffer = dimensionPage.getEncodedData();
            int bufferSize = buffer.limit();
            currentOffsetInFile += fileChannel.write(buffer);
            offset += bufferSize;
        }
    }
    dimensionOffset = offset;
    int dataChunkStartIndex = encodedBlocklet.getNumberOfDimension();
    for (int i = 0; i < numberOfMeasures; i++) {
        currentDataChunksOffset.add(offset);
        currentDataChunksLength.add(dataChunkBytes[dataChunkStartIndex].length);
        buffer = ByteBuffer.wrap(dataChunkBytes[dataChunkStartIndex]);
        currentOffsetInFile += fileChannel.write(buffer);
        offset += dataChunkBytes[dataChunkStartIndex].length;
        dataChunkStartIndex++;
        BlockletEncodedColumnPage blockletEncodedColumnPage = encodedBlocklet.getEncodedMeasureColumnPages().get(i);
        for (EncodedColumnPage measurePage : blockletEncodedColumnPage.getEncodedColumnPageList()) {
            buffer = measurePage.getEncodedData();
            int bufferSize = buffer.limit();
            currentOffsetInFile += fileChannel.write(buffer);
            offset += bufferSize;
        }
    }
    measureOffset = offset;
    blockletIndex.add(CarbonMetadataUtil.getBlockletIndex(encodedBlocklet, model.getSegmentProperties().getMeasures()));
    BlockletInfo3 blockletInfo3 = new BlockletInfo3(encodedBlocklet.getBlockletSize(), currentDataChunksOffset, currentDataChunksLength, dimensionOffset, measureOffset, encodedBlocklet.getNumberOfPages());
    // Avoid storing as integer in encodedBocklet,
    // but in thrift store as int for large number of rows future support
    List<Integer> rowList = new ArrayList<>(encodedBlocklet.getRowCountInPage().size());
    for (int rows : encodedBlocklet.getRowCountInPage()) {
        rowList.add(rows);
    }
    blockletInfo3.setRow_count_in_page(rowList);
    blockletMetadata.add(blockletInfo3);
}

Example 3 with BlockletEncodedColumnPage

use of org.apache.carbondata.core.datastore.blocklet.BlockletEncodedColumnPage in project carbondata by apache.

the class CarbonMetadataUtil method getDimensionDataChunk3.

/**
 * return DataChunk3 for the dimension column (specified by `columnIndex`)
 * in `encodedTablePageList`
 */
public static DataChunk3 getDimensionDataChunk3(EncodedBlocklet encodedBlocklet, int columnIndex) {
    List<DataChunk2> dataChunksList = new ArrayList<>();
    BlockletEncodedColumnPage blockletEncodedColumnPage = encodedBlocklet.getEncodedDimensionColumnPages().get(columnIndex);
    for (EncodedColumnPage encodedColumnPage : blockletEncodedColumnPage.getEncodedColumnPageList()) {
        dataChunksList.add(encodedColumnPage.getPageMetadata());
    }
    return CarbonMetadataUtil.getDataChunk3(dataChunksList, blockletEncodedColumnPage.getEncodedDictionary());
}