Examples with BlockletInfo org.apache.carbondata.core.metadata.blocklet.BlockletInfo used on opensource projects

Example 6 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class DataFileFooterConverter2 method readDataFileFooter.

/**
 * Below method will be used to convert thrift file meta to wrapper file meta
 */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    CarbonFooterReader reader = new CarbonFooterReader(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
    FileFooter footer = reader.readFooter();
    dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) footer.getVersion()));
    dataFileFooter.setNumberOfRows(footer.getNum_rows());
    dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<org.apache.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
    for (int i = 0; i < table_columns.size(); i++) {
        columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
    }
    dataFileFooter.setColumnInTable(columnSchemaList);
    List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
    List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
    for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
        BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
        blockletIndexList.add(blockletIndex);
    }
    List<org.apache.carbondata.format.BlockletInfo2> leaf_node_infos_Thrift = footer.getBlocklet_info_list2();
    List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
    for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
        BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i), getNumberOfDimensionColumns(columnSchemaList));
        blockletInfo.setBlockletIndex(blockletIndexList.get(i));
        blockletInfoList.add(blockletInfo);
    }
    dataFileFooter.setBlockletList(blockletInfoList);
    dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    return dataFileFooter;
}

Example 7 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class DataFileFooterConverterV3 method readDataFileFooter.

/**
 * Below method will be used to convert thrift file meta to wrapper file meta
 * This method will read the footer from footer offset present in the data file
 * 1. It will read the header from carbon data file, header starts from 0 offset
 * 2. It will set the stream offset
 * 3. It will read the footer data from file
 * 4. parse the footer to thrift object
 * 5. convert to wrapper object
 *
 * @param tableBlockInfo
 *        table block info
 * @return data file footer
 */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(tableBlockInfo.getFilePath());
    FileHeader fileHeader = carbonHeaderReader.readHeader();
    CarbonFooterReaderV3 reader = new CarbonFooterReaderV3(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
    FileFooter3 footer = reader.readFooterVersion3();
    dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) fileHeader.getVersion()));
    dataFileFooter.setNumberOfRows(footer.getNum_rows());
    dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
    dataFileFooter.setSchemaUpdatedTimeStamp(fileHeader.getTime_stamp());
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
    for (int i = 0; i < table_columns.size(); i++) {
        columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
    }
    dataFileFooter.setColumnInTable(columnSchemaList);
    List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
    List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
    for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
        BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
        blockletIndexList.add(blockletIndex);
    }
    List<org.apache.carbondata.format.BlockletInfo3> leaf_node_infos_Thrift = footer.getBlocklet_info_list3();
    List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
    for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
        BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i), CarbonUtil.getNumberOfDimensionColumns(columnSchemaList));
        blockletInfo.setBlockletIndex(blockletIndexList.get(i));
        blockletInfoList.add(blockletInfo);
    }
    dataFileFooter.setBlockletList(blockletInfoList);
    dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    return dataFileFooter;
}

Example 8 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class DataFileFooterConverterV3 method getBlockletInfo.

/**
 * Below method is to convert the blocklet info of the thrift to wrapper
 * blocklet info
 *
 * @param blockletInfoThrift blocklet info of the thrift
 * @return blocklet info wrapper
 */
public BlockletInfo getBlockletInfo(org.apache.carbondata.format.BlockletInfo3 blockletInfoThrift, int numberOfDimensionColumns) {
    BlockletInfo blockletInfo = new BlockletInfo();
    List<Long> dimensionColumnChunkOffsets = blockletInfoThrift.getColumn_data_chunks_offsets().subList(0, numberOfDimensionColumns);
    List<Long> measureColumnChunksOffsets = blockletInfoThrift.getColumn_data_chunks_offsets().subList(numberOfDimensionColumns, blockletInfoThrift.getColumn_data_chunks_offsets().size());
    List<Integer> dimensionColumnChunkLength = blockletInfoThrift.getColumn_data_chunks_length().subList(0, numberOfDimensionColumns);
    List<Integer> measureColumnChunksLength = blockletInfoThrift.getColumn_data_chunks_length().subList(numberOfDimensionColumns, blockletInfoThrift.getColumn_data_chunks_offsets().size());
    blockletInfo.setDimensionChunkOffsets(dimensionColumnChunkOffsets);
    blockletInfo.setMeasureChunkOffsets(measureColumnChunksOffsets);
    blockletInfo.setDimensionChunksLength(dimensionColumnChunkLength);
    blockletInfo.setMeasureChunksLength(measureColumnChunksLength);
    blockletInfo.setNumberOfRows(blockletInfoThrift.getNum_rows());
    blockletInfo.setDimensionOffset(blockletInfoThrift.getDimension_offsets());
    blockletInfo.setMeasureOffsets(blockletInfoThrift.getMeasure_offsets());
    blockletInfo.setNumberOfPages(blockletInfoThrift.getNumber_number_of_pages());
    return blockletInfo;
}

Example 9 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class AbstractDataFileFooterConverter method getIndexInfo.

/**
 * Below method will be used to get the index info from index file
 *
 * @param filePath           file path of the index file
 * @return list of index info
 * @throws IOException problem while reading the index file
 */
public List<DataFileFooter> getIndexInfo(String filePath, byte[] fileData) throws IOException {
    CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
    List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();
    String parentPath = filePath.substring(0, filePath.lastIndexOf("/"));
    try {
        // open the reader
        if (fileData != null) {
            indexReader.openThriftReader(fileData);
        } else {
            indexReader.openThriftReader(filePath);
        }
        // get the index header
        org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
        List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
        List<org.apache.carbondata.format.ColumnSchema> table_columns = readIndexHeader.getTable_columns();
        for (int i = 0; i < table_columns.size(); i++) {
            columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
        }
        // get the segment info
        SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());
        BlockletIndex blockletIndex = null;
        DataFileFooter dataFileFooter = null;
        // read the block info from file
        while (indexReader.hasNext()) {
            BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();
            blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());
            dataFileFooter = new DataFileFooter();
            TableBlockInfo tableBlockInfo = getTableBlockInfo(readBlockIndexInfo, readIndexHeader, parentPath);
            dataFileFooter.setBlockletIndex(blockletIndex);
            dataFileFooter.setColumnInTable(columnSchemaList);
            dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());
            dataFileFooter.setBlockInfo(new BlockInfo(tableBlockInfo));
            dataFileFooter.setSegmentInfo(segmentInfo);
            dataFileFooter.setVersionId(tableBlockInfo.getVersion());
            // In case of old schema time stamp will not be found in the index header
            if (readIndexHeader.isSetSchema_time_stamp()) {
                dataFileFooter.setSchemaUpdatedTimeStamp(readIndexHeader.getSchema_time_stamp());
            }
            if (readBlockIndexInfo.isSetBlocklet_info()) {
                List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
                BlockletInfo blockletInfo = new DataFileFooterConverterV3().getBlockletInfo(readBlockIndexInfo.getBlocklet_info(), CarbonUtil.getNumberOfDimensionColumns(columnSchemaList));
                blockletInfo.setBlockletIndex(blockletIndex);
                blockletInfoList.add(blockletInfo);
                dataFileFooter.setBlockletList(blockletInfoList);
            }
            dataFileFooters.add(dataFileFooter);
        }
    } finally {
        indexReader.closeThriftReader();
    }
    return dataFileFooters;
}

Example 10 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class BlockletDetailInfo method readFields.

@Override
public void readFields(DataInput in) throws IOException {
    rowCount = in.readInt();
    pagesCount = in.readShort();
    versionNumber = in.readShort();
    blockletId = in.readShort();
    dimLens = new int[in.readShort()];
    for (int i = 0; i < dimLens.length; i++) {
        dimLens[i] = in.readInt();
    }
    schemaUpdatedTimeStamp = in.readLong();
    if (in.readBoolean()) {
        blockletInfo = new BlockletInfo();
        blockletInfo.readFields(in);
    }
    blockFooterOffset = in.readLong();
    int bytesSize = in.readInt();
    byte[] schemaArray = new byte[bytesSize];
    in.readFully(schemaArray);
    readColumnSchema(schemaArray);
    blockSize = in.readLong();
}