Examples with BlockletInfo org.apache.carbondata.core.metadata.blocklet.BlockletInfo used on opensource projects

Example 1 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class SegmentTaskIndexTest method setUp.

@BeforeClass
public static void setUp() {
    segmentInfo = new SegmentInfo();
    footer = new DataFileFooter();
    columnSchema = new ColumnSchema();
    blockletInfo = new BlockletInfo();
    blockletIndex = new BlockletIndex();
}

Example 2 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class BlockIndexTest method setUp.

@BeforeClass
public static void setUp() {
    segmentInfo = new SegmentInfo();
    footer = new DataFileFooter();
    columnSchema = new ColumnSchema();
    blockletInfo = new BlockletInfo();
    blockletIndex = new BlockletIndex();
}

Example 3 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class AbstractQueryExecutor method readAndFillBlockletInfo.

/**
 * Read the file footer of block file and get the blocklets to query
 */
private void readAndFillBlockletInfo(TableBlockInfo blockInfo, List<TableBlockInfo> tableBlockInfos, BlockletDetailInfo blockletDetailInfo) throws IOException {
    blockInfo.setBlockOffset(blockletDetailInfo.getBlockFooterOffset());
    blockInfo.setDetailInfo(null);
    DataFileFooter fileFooter = CarbonUtil.readMetadatFile(blockInfo);
    blockInfo.setDetailInfo(blockletDetailInfo);
    List<BlockletInfo> blockletList = fileFooter.getBlockletList();
    short count = 0;
    for (BlockletInfo blockletInfo : blockletList) {
        TableBlockInfo info = blockInfo.copy();
        BlockletDetailInfo detailInfo = info.getDetailInfo();
        detailInfo.setRowCount(blockletInfo.getNumberOfRows());
        // update min and max values in case of old store for measures as min and max is written
        // opposite for measures in old store
        byte[][] maxValues = CarbonUtil.updateMinMaxValues(fileFooter, blockletInfo.getBlockletIndex().getMinMaxIndex().getMaxValues(), blockletInfo.getBlockletIndex().getMinMaxIndex().getMinValues(), false);
        byte[][] minValues = CarbonUtil.updateMinMaxValues(fileFooter, blockletInfo.getBlockletIndex().getMinMaxIndex().getMaxValues(), blockletInfo.getBlockletIndex().getMinMaxIndex().getMinValues(), true);
        blockletInfo.getBlockletIndex().getMinMaxIndex().setMaxValues(maxValues);
        blockletInfo.getBlockletIndex().getMinMaxIndex().setMinValues(minValues);
        detailInfo.setBlockletInfo(blockletInfo);
        detailInfo.setPagesCount((short) blockletInfo.getNumberOfPages());
        detailInfo.setBlockletId(count);
        info.setDataBlockFromOldStore(true);
        tableBlockInfos.add(info);
        count++;
    }
}

Example 4 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class DataFileFooterConverter method readDataFileFooter.

/**
 * Below method will be used to convert thrift file meta to wrapper file meta
 */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    FileReader fileReader = null;
    try {
        long completeBlockLength = tableBlockInfo.getBlockLength();
        long footerPointer = completeBlockLength - 8;
        fileReader = FileFactory.getFileHolder(FileFactory.getFileType(tableBlockInfo.getFilePath()));
        long actualFooterOffset = fileReader.readLong(tableBlockInfo.getFilePath(), footerPointer);
        CarbonFooterReader reader = new CarbonFooterReader(tableBlockInfo.getFilePath(), actualFooterOffset);
        FileFooter footer = reader.readFooter();
        dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) footer.getVersion()));
        dataFileFooter.setNumberOfRows(footer.getNum_rows());
        dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
        List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
        List<org.apache.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
        for (int i = 0; i < table_columns.size(); i++) {
            columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
        }
        dataFileFooter.setColumnInTable(columnSchemaList);
        List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
        List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
        for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
            BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
            blockletIndexList.add(blockletIndex);
        }
        List<org.apache.carbondata.format.BlockletInfo> leaf_node_infos_Thrift = footer.getBlocklet_info_list();
        List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
        for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
            BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i));
            blockletInfo.setBlockletIndex(blockletIndexList.get(i));
            blockletInfoList.add(blockletInfo);
        }
        dataFileFooter.setBlockletList(blockletInfoList);
        dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    } finally {
        if (null != fileReader) {
            fileReader.finish();
        }
    }
    return dataFileFooter;
}

Example 5 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class DataFileFooterConverter method getBlockletInfo.

/**
 * Below method is to convert the blocklet info of the thrift to wrapper
 * blocklet info
 *
 * @param blockletInfoThrift blocklet info of the thrift
 * @return blocklet info wrapper
 */
private BlockletInfo getBlockletInfo(org.apache.carbondata.format.BlockletInfo blockletInfoThrift) {
    BlockletInfo blockletInfo = new BlockletInfo();
    List<DataChunk> dimensionColumnChunk = new ArrayList<DataChunk>();
    List<DataChunk> measureChunk = new ArrayList<DataChunk>();
    Iterator<org.apache.carbondata.format.DataChunk> column_data_chunksIterator = blockletInfoThrift.getColumn_data_chunksIterator();
    if (null != column_data_chunksIterator) {
        while (column_data_chunksIterator.hasNext()) {
            org.apache.carbondata.format.DataChunk next = column_data_chunksIterator.next();
            if (next.isRowMajor()) {
                dimensionColumnChunk.add(getDataChunk(next, false));
            } else if (next.getEncoders().contains(org.apache.carbondata.format.Encoding.DELTA)) {
                measureChunk.add(getDataChunk(next, true));
            } else {
                dimensionColumnChunk.add(getDataChunk(next, false));
            }
        }
    }
    blockletInfo.setDimensionColumnChunk(dimensionColumnChunk);
    blockletInfo.setMeasureColumnChunk(measureChunk);
    blockletInfo.setNumberOfRows(blockletInfoThrift.getNum_rows());
    return blockletInfo;
}