Examples with BlockMetaInfo org.apache.carbondata.core.indexstore.BlockMetaInfo used on opensource projects

Example 1 with BlockMetaInfo

use of org.apache.carbondata.core.indexstore.BlockMetaInfo in project carbondata by apache.

the class BlockletDataMap method init.

@Override
public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
    long startTime = System.currentTimeMillis();
    assert (dataMapModel instanceof BlockletDataMapModel);
    BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel) dataMapModel;
    DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
    List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
    Path path = new Path(blockletDataMapInfo.getFilePath());
    byte[] filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    byte[] segmentId = blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    DataMapRowImpl summaryRow = null;
    byte[] schemaBinary = null;
    // below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
    // is id assigned to a blocklet within a part file
    String tempFilePath = null;
    int relativeBlockletId = 0;
    for (DataFileFooter fileFooter : indexInfo) {
        if (segmentProperties == null) {
            List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
            schemaBinary = convertSchemaToBinary(columnInTable);
            columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
            segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
            createSchema(segmentProperties);
            createSummarySchema(segmentProperties, schemaBinary, filePath, fileName, segmentId);
        }
        TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
        BlockMetaInfo blockMetaInfo = blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
        // the file exists physically or not
        if (blockMetaInfo != null) {
            if (fileFooter.getBlockletList() == null) {
                // This is old store scenario, here blocklet information is not available in index file so
                // load only block info
                summaryRow = loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow, blockMetaInfo);
            } else {
                // blocklet ID will start from 0 again only when part file path is changed
                if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
                    tempFilePath = blockInfo.getFilePath();
                    relativeBlockletId = 0;
                }
                summaryRow = loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow, blockMetaInfo, relativeBlockletId);
                // this is done because relative blocklet id need to be incremented based on the
                // total number of blocklets
                relativeBlockletId += fileFooter.getBlockletList().size();
            }
        }
    }
    if (unsafeMemoryDMStore != null) {
        unsafeMemoryDMStore.finishWriting();
    }
    if (null != unsafeMemorySummaryDMStore) {
        addTaskSummaryRowToUnsafeMemoryStore(summaryRow, schemaBinary, filePath, fileName, segmentId);
        unsafeMemorySummaryDMStore.finishWriting();
    }
    LOGGER.info("Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + "is " + (System.currentTimeMillis() - startTime));
}