Examples with BlockColumnMetaDataInfo org.apache.carbondata.core.segmentmeta.BlockColumnMetaDataInfo used on opensource projects

Example 1 with BlockColumnMetaDataInfo

use of org.apache.carbondata.core.segmentmeta.BlockColumnMetaDataInfo in project carbondata by apache.

the class AbstractFactDataWriter method writeIndexFile.

/**
 * Below method will be used to write the idex file
 *
 * @throws IOException               throws io exception if any problem while writing
 * @throws CarbonDataWriterException data writing
 */
protected void writeIndexFile() throws IOException, CarbonDataWriterException {
    if (blockIndexInfoList.size() == 0) {
        // no need to write index file, if data file is not there.
        return;
    }
    // get the header
    IndexHeader indexHeader = CarbonMetadataUtil.getIndexHeader(thriftColumnSchemaList, model.getBucketId(), model.getSchemaUpdatedTimeStamp());
    indexHeader.setIs_sort(model.getSortScope() != null && model.getSortScope() != NO_SORT);
    // get the block index info thrift
    List<BlockIndex> blockIndexThrift = CarbonMetadataUtil.getBlockIndexInfo(blockIndexInfoList);
    // get all block minmax and add to segmentMinMaxMap
    CarbonTable carbonTable = model.getTableSpec().getCarbonTable();
    if (null != model.getSegmentId() && !carbonTable.isHivePartitionTable() && !carbonTable.isIndexTable()) {
        for (BlockIndexInfo blockIndex : blockIndexInfoList) {
            byte[][] min = blockIndex.getBlockletIndex().getMinMaxIndex().getMinValues();
            byte[][] max = blockIndex.getBlockletIndex().getMinMaxIndex().getMaxValues();
            BlockColumnMetaDataInfo blockColumnMetaDataInfo = new BlockColumnMetaDataInfo(thriftColumnSchemaList, min, max);
            SegmentMetaDataInfoStats.getInstance().setBlockMetaDataInfo(model.getTableName(), model.getSegmentId(), blockColumnMetaDataInfo, this.model.getWrapperColumnSchema());
        }
    }
    String indexFileName;
    if (enableDirectlyWriteDataToStorePath) {
        String rawFileName = model.getCarbonDataDirectoryPath() + CarbonCommonConstants.FILE_SEPARATOR + CarbonTablePath.getCarbonIndexFileName(model.getCarbonDataFileAttributes().getTaskId(), model.getBucketId(), model.getTaskExtension(), "" + model.getCarbonDataFileAttributes().getFactTimeStamp(), model.getSegmentId());
        indexFileName = FileFactory.getUpdatedFilePath(rawFileName);
    } else {
        // randomly choose a temp location for index file
        String[] tempLocations = model.getStoreLocation();
        String chosenTempLocation = tempLocations[new Random().nextInt(tempLocations.length)];
        LOGGER.info("Randomly choose index file location: " + chosenTempLocation);
        indexFileName = chosenTempLocation + File.separator + CarbonTablePath.getCarbonIndexFileName(model.getCarbonDataFileAttributes().getTaskId(), model.getBucketId(), model.getTaskExtension(), "" + model.getCarbonDataFileAttributes().getFactTimeStamp(), model.getSegmentId());
    }
    CarbonIndexFileWriter writer = new CarbonIndexFileWriter();
    // open file
    writer.openThriftWriter(indexFileName);
    // write the header first
    writer.writeThrift(indexHeader);
    // write the indexes
    for (BlockIndex blockIndex : blockIndexThrift) {
        writer.writeThrift(blockIndex);
    }
    writer.close();
    if (!enableDirectlyWriteDataToStorePath) {
        CarbonUtil.copyCarbonDataFileToCarbonStorePath(indexFileName, model.getCarbonDataDirectoryPath(), fileSizeInBytes, metrics);
        FileFactory.deleteFile(indexFileName);
    } else if (model.getTableSpec().getCarbonTable().isHivePartitionTable() && model.getCarbonDataDirectoryPath().endsWith(".tmp")) {
        if (metrics != null) {
            addOutputFilesInfoToMetrics(indexFileName);
        }
    }
}