Examples with BlockIndex org.apache.carbondata.format.BlockIndex used on opensource projects

Example 1 with BlockIndex

use of org.apache.carbondata.format.BlockIndex in project carbondata by apache.

the class StreamSegment method updateIndexFile.

/**
 * update carbonindex file after a stream batch.
 */
public static void updateIndexFile(String segmentDir) throws IOException {
    FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
    String filePath = CarbonTablePath.getCarbonStreamIndexFilePath(segmentDir);
    String tempFilePath = filePath + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
    CarbonIndexFileWriter writer = new CarbonIndexFileWriter();
    try {
        writer.openThriftWriter(tempFilePath);
        CarbonFile[] files = listDataFiles(segmentDir, fileType);
        BlockIndex blockIndex;
        for (CarbonFile file : files) {
            blockIndex = new BlockIndex();
            blockIndex.setFile_name(file.getName());
            blockIndex.setFile_size(file.getSize());
            // TODO need to collect these information
            blockIndex.setNum_rows(-1);
            blockIndex.setOffset(-1);
            blockIndex.setBlock_index(new BlockletIndex());
            writer.writeThrift(blockIndex);
        }
        writer.close();
        CarbonFile tempFile = FileFactory.getCarbonFile(tempFilePath, fileType);
        if (!tempFile.renameForce(filePath)) {
            throw new IOException("temporary file renaming failed, src=" + tempFilePath + ", dest=" + filePath);
        }
    } catch (IOException ex) {
        try {
            writer.close();
        } catch (IOException t) {
            LOGGER.error(t);
        }
        throw ex;
    }
}

Example 2 with BlockIndex

use of org.apache.carbondata.format.BlockIndex in project carbondata by apache.

the class StreamSegment method size.

/**
 * calculate the size of the segment by the accumulation of data sizes in index file
 */
public static long size(String segmentDir) throws IOException {
    long size = 0;
    FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
    if (FileFactory.isFileExist(segmentDir, fileType)) {
        String indexPath = CarbonTablePath.getCarbonStreamIndexFilePath(segmentDir);
        CarbonFile index = FileFactory.getCarbonFile(indexPath, fileType);
        if (index.exists()) {
            CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
            try {
                indexReader.openThriftReader(indexPath);
                while (indexReader.hasNext()) {
                    BlockIndex blockIndex = indexReader.readBlockIndexInfo();
                    size += blockIndex.getFile_size();
                }
            } finally {
                indexReader.closeThriftReader();
            }
        }
    }
    return size;
}

Example 3 with BlockIndex

use of org.apache.carbondata.format.BlockIndex in project carbondata by apache.

the class StreamSegment method recoverSegmentIfRequired.

/**
 * check the health of stream segment and try to recover segment from job fault
 * this method will be invoked in following scenarios.
 * 1. at the begin of the streaming (StreamSinkFactory.getStreamSegmentId)
 * 2. after job failed (CarbonAppendableStreamSink.writeDataFileJob)
 */
public static void recoverSegmentIfRequired(String segmentDir) throws IOException {
    FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
    if (FileFactory.isFileExist(segmentDir, fileType)) {
        String indexName = CarbonTablePath.getCarbonStreamIndexFileName();
        String indexPath = segmentDir + File.separator + indexName;
        CarbonFile index = FileFactory.getCarbonFile(indexPath, fileType);
        CarbonFile[] files = listDataFiles(segmentDir, fileType);
        // index file exists
        if (index.exists()) {
            // data file exists
            if (files.length > 0) {
                CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
                try {
                    // map block index
                    indexReader.openThriftReader(indexPath);
                    Map<String, Long> tableSizeMap = new HashMap<>();
                    while (indexReader.hasNext()) {
                        BlockIndex blockIndex = indexReader.readBlockIndexInfo();
                        tableSizeMap.put(blockIndex.getFile_name(), blockIndex.getFile_size());
                    }
                    // recover each file
                    for (CarbonFile file : files) {
                        Long size = tableSizeMap.get(file.getName());
                        if (null == size || size == 0) {
                            file.delete();
                        } else if (size < file.getSize()) {
                            FileFactory.truncateFile(file.getCanonicalPath(), fileType, size);
                        }
                    }
                } finally {
                    indexReader.closeThriftReader();
                }
            }
        } else {
            if (files.length > 0) {
                for (CarbonFile file : files) {
                    file.delete();
                }
            }
        }
    }
}

Example 4 with BlockIndex

use of org.apache.carbondata.format.BlockIndex in project carbondata by apache.

the class StreamSegment method recoverFileIfRequired.

/**
 * check the health of stream data file and try to recover data file from task fault
 *  this method will be invoked in following scenarios.
 *  1. at the begin of writing data file task
 */
public static void recoverFileIfRequired(String segmentDir, String fileName, String indexName) throws IOException {
    FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
    String filePath = segmentDir + File.separator + fileName;
    CarbonFile file = FileFactory.getCarbonFile(filePath, fileType);
    String indexPath = segmentDir + File.separator + indexName;
    CarbonFile index = FileFactory.getCarbonFile(indexPath, fileType);
    if (file.exists() && index.exists()) {
        CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
        try {
            indexReader.openThriftReader(indexPath);
            while (indexReader.hasNext()) {
                BlockIndex blockIndex = indexReader.readBlockIndexInfo();
                if (blockIndex.getFile_name().equals(fileName)) {
                    if (blockIndex.getFile_size() == 0) {
                        file.delete();
                    } else if (blockIndex.getFile_size() < file.getSize()) {
                        FileFactory.truncateFile(filePath, fileType, blockIndex.getFile_size());
                    }
                }
            }
        } finally {
            indexReader.closeThriftReader();
        }
    }
}

Example 5 with BlockIndex

use of org.apache.carbondata.format.BlockIndex in project carbondata by apache.

the class CarbonTableInputFormat method getSplitsOfStreaming.

/**
 * use file list in .carbonindex file to get the split of streaming.
 */
public List<InputSplit> getSplitsOfStreaming(JobContext job, AbsoluteTableIdentifier identifier, List<Segment> streamSegments) throws IOException {
    List<InputSplit> splits = new ArrayList<InputSplit>();
    if (streamSegments != null && !streamSegments.isEmpty()) {
        numStreamSegments = streamSegments.size();
        long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
        long maxSize = getMaxSplitSize(job);
        for (Segment segment : streamSegments) {
            String segmentDir = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo());
            FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
            if (FileFactory.isFileExist(segmentDir, fileType)) {
                String indexName = CarbonTablePath.getCarbonStreamIndexFileName();
                String indexPath = segmentDir + File.separator + indexName;
                CarbonFile index = FileFactory.getCarbonFile(indexPath, fileType);
                // index file exists
                if (index.exists()) {
                    // data file exists
                    CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
                    try {
                        // map block index
                        indexReader.openThriftReader(indexPath);
                        while (indexReader.hasNext()) {
                            BlockIndex blockIndex = indexReader.readBlockIndexInfo();
                            String filePath = segmentDir + File.separator + blockIndex.getFile_name();
                            Path path = new Path(filePath);
                            long length = blockIndex.getFile_size();
                            if (length != 0) {
                                BlockLocation[] blkLocations;
                                FileSystem fs = FileFactory.getFileSystem(path);
                                FileStatus file = fs.getFileStatus(path);
                                blkLocations = fs.getFileBlockLocations(path, 0, length);
                                long blockSize = file.getBlockSize();
                                long splitSize = computeSplitSize(blockSize, minSize, maxSize);
                                long bytesRemaining = length;
                                while (((double) bytesRemaining) / splitSize > 1.1) {
                                    int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
                                    splits.add(makeSplit(segment.getSegmentNo(), path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts(), FileFormat.ROW_V1));
                                    bytesRemaining -= splitSize;
                                }
                                if (bytesRemaining != 0) {
                                    int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
                                    splits.add(makeSplit(segment.getSegmentNo(), path, length - bytesRemaining, bytesRemaining, blkLocations[blkIndex].getHosts(), blkLocations[blkIndex].getCachedHosts(), FileFormat.ROW_V1));
                                }
                            } else {
                                // Create empty hosts array for zero length files
                                splits.add(makeSplit(segment.getSegmentNo(), path, 0, length, new String[0], FileFormat.ROW_V1));
                            }
                        }
                    } finally {
                        indexReader.closeThriftReader();
                    }
                }
            }
        }
    }
    return splits;
}