Examples with BlockletInfo org.apache.carbondata.core.metadata.blocklet.BlockletInfo used on opensource projects

Example 16 with BlockletInfo

use of org.apache.carbondata.core.metadata.blocklet.BlockletInfo in project carbondata by apache.

the class SegmentIndexFileStore method readIndexAndFillBlockletInfo.

/**
 * This method will read the index information from carbon index file
 *
 * @param indexFile
 * @return
 * @throws IOException
 */
private void readIndexAndFillBlockletInfo(CarbonFile indexFile) throws IOException {
    // flag to take decision whether carbondata file footer reading is required.
    // If the index file does not contain the file footer then carbondata file footer
    // read is required else not required
    boolean isCarbonDataFileFooterReadRequired = true;
    List<BlockletInfo> blockletInfoList = null;
    List<BlockIndex> blockIndexThrift = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
    try {
        indexReader.openThriftReader(indexFile.getCanonicalPath());
        // get the index header
        org.apache.carbondata.format.IndexHeader indexHeader = indexReader.readIndexHeader();
        DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
        String filePath = indexFile.getCanonicalPath();
        String parentPath = filePath.substring(0, filePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR));
        while (indexReader.hasNext()) {
            BlockIndex blockIndex = indexReader.readBlockIndexInfo();
            if (blockIndex.isSetBlocklet_info()) {
                // this case will come in case segment index compaction property is set to false from the
                // application and alter table segment index compaction is run manually. In that case
                // blocklet info will be present in the index but read carbon data file footer property
                // will be true
                isCarbonDataFileFooterReadRequired = false;
                break;
            } else {
                TableBlockInfo blockInfo = fileFooterConverter.getTableBlockInfo(blockIndex, indexHeader, parentPath);
                blockletInfoList = getBlockletInfoFromIndexInfo(blockInfo);
            }
            // the same entry with different blocklet info need to be repeated
            for (int i = 0; i < blockletInfoList.size(); i++) {
                BlockIndex blockIndexReplica = blockIndex.deepCopy();
                BlockletInfo blockletInfo = blockletInfoList.get(i);
                blockIndexReplica.setBlock_index(CarbonMetadataUtil.getBlockletIndex(blockletInfo.getBlockletIndex()));
                blockIndexReplica.setBlocklet_info(CarbonMetadataUtil.getBlocletInfo3(blockletInfo));
                blockIndexThrift.add(blockIndexReplica);
            }
        }
        // read complete file at once
        if (!isCarbonDataFileFooterReadRequired) {
            readIndexFile(indexFile);
        } else {
            int totalSize = 0;
            List<byte[]> blockIndexByteArrayList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
            byte[] indexHeaderBytes = CarbonUtil.getByteArray(indexHeader);
            totalSize += indexHeaderBytes.length;
            blockIndexByteArrayList.add(indexHeaderBytes);
            for (BlockIndex blockIndex : blockIndexThrift) {
                byte[] indexInfoBytes = CarbonUtil.getByteArray(blockIndex);
                totalSize += indexInfoBytes.length;
                blockIndexByteArrayList.add(indexInfoBytes);
            }
            ByteBuffer byteBuffer = ByteBuffer.allocate(totalSize);
            for (byte[] blockIndexBytes : blockIndexByteArrayList) {
                byteBuffer.put(blockIndexBytes);
            }
            carbonIndexMap.put(indexFile.getName(), byteBuffer.array());
        }
    } finally {
        indexReader.closeThriftReader();
    }
}