Examples with CarbonIndexFileReader org.apache.carbondata.core.reader.CarbonIndexFileReader used on opensource projects

Example 6 with CarbonIndexFileReader

use of org.apache.carbondata.core.reader.CarbonIndexFileReader in project carbondata by apache.

the class StreamSegment method recoverFileIfRequired.

/**
 * check the health of stream data file and try to recover data file from task fault
 *  this method will be invoked in following scenarios.
 *  1. at the begin of writing data file task
 */
public static void recoverFileIfRequired(String segmentDir, String fileName, String indexName) throws IOException {
    String filePath = segmentDir + File.separator + fileName;
    CarbonFile file = FileFactory.getCarbonFile(filePath);
    String indexPath = segmentDir + File.separator + indexName;
    CarbonFile index = FileFactory.getCarbonFile(indexPath);
    if (file.exists() && index.exists()) {
        CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
        try {
            indexReader.openThriftReader(indexPath);
            while (indexReader.hasNext()) {
                BlockIndex blockIndex = indexReader.readBlockIndexInfo();
                if (blockIndex.getFile_name().equals(fileName)) {
                    if (blockIndex.getFile_size() == 0) {
                        file.delete();
                    } else if (blockIndex.getFile_size() < file.getSize()) {
                        FileFactory.truncateFile(filePath, blockIndex.getFile_size());
                    }
                    break;
                }
            }
        } finally {
            indexReader.closeThriftReader();
        }
    }
}

Example 7 with CarbonIndexFileReader

use of org.apache.carbondata.core.reader.CarbonIndexFileReader in project carbondata by apache.

the class StreamSegment method readIndexFile.

/**
 * read index file to list BlockIndex
 *
 * @param indexPath path of the index file
 * @return the list of BlockIndex in the index file
 * @throws IOException failed to read index file
 */
public static List<BlockIndex> readIndexFile(String indexPath) throws IOException {
    List<BlockIndex> blockIndexList = new ArrayList<>();
    CarbonFile index = FileFactory.getCarbonFile(indexPath);
    if (index.exists()) {
        CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
        try {
            indexReader.openThriftReader(indexPath);
            while (indexReader.hasNext()) {
                blockIndexList.add(indexReader.readBlockIndexInfo());
            }
        } finally {
            indexReader.closeThriftReader();
        }
    }
    return blockIndexList;
}

Example 8 with CarbonIndexFileReader

use of org.apache.carbondata.core.reader.CarbonIndexFileReader in project carbondata by apache.

the class StreamSegment method recoverSegmentIfRequired.

/**
 * check the health of stream segment and try to recover segment from job fault
 * this method will be invoked in following scenarios.
 * 1. at the begin of the streaming (StreamSinkFactory.getStreamSegmentId)
 * 2. after job failed (CarbonAppendableStreamSink.writeDataFileJob)
 */
public static void recoverSegmentIfRequired(String segmentDir) throws IOException {
    if (FileFactory.isFileExist(segmentDir)) {
        String indexName = CarbonTablePath.getCarbonStreamIndexFileName();
        String indexPath = segmentDir + File.separator + indexName;
        CarbonFile index = FileFactory.getCarbonFile(indexPath);
        CarbonFile[] files = listDataFiles(segmentDir);
        // index file exists
        if (index.exists()) {
            // data file exists
            if (files.length > 0) {
                CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
                try {
                    // map block index
                    indexReader.openThriftReader(indexPath);
                    Map<String, Long> tableSizeMap = new HashMap<>();
                    while (indexReader.hasNext()) {
                        BlockIndex blockIndex = indexReader.readBlockIndexInfo();
                        tableSizeMap.put(blockIndex.getFile_name(), blockIndex.getFile_size());
                    }
                    // recover each file
                    for (CarbonFile file : files) {
                        Long size = tableSizeMap.get(file.getName());
                        if (null == size || size == 0) {
                            file.delete();
                        } else if (size < file.getSize()) {
                            FileFactory.truncateFile(file.getCanonicalPath(), size);
                        }
                    }
                } finally {
                    indexReader.closeThriftReader();
                }
            }
        } else {
            if (files.length > 0) {
                for (CarbonFile file : files) {
                    file.delete();
                }
            }
        }
    }
}

Example 9 with CarbonIndexFileReader

use of org.apache.carbondata.core.reader.CarbonIndexFileReader in project carbondata by apache.

the class SegmentIndexFileStore method readIndexHeader.

public static IndexHeader readIndexHeader(String indexFilePath, Configuration configuration) {
    byte[] indexContent = null;
    if (indexFilePath.toLowerCase().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
        SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore(configuration);
        try {
            indexFileStore.readMergeFile(indexFilePath);
        } catch (IOException ex) {
            LOGGER.error(ex);
        }
        Iterator<Map.Entry<String, byte[]>> iterator = indexFileStore.getCarbonIndexMap().entrySet().iterator();
        if (iterator.hasNext()) {
            indexContent = iterator.next().getValue();
        }
    }
    CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
    IndexHeader indexHeader = null;
    try {
        if (indexContent == null) {
            indexReader.openThriftReader(indexFilePath);
        } else {
            indexReader.openThriftReader(indexContent);
        }
        // get the index header
        indexHeader = indexReader.readIndexHeader();
    } catch (IOException ex) {
        LOGGER.error(ex);
    } finally {
        indexReader.closeThriftReader();
    }
    return indexHeader;
}

Example 10 with CarbonIndexFileReader

use of org.apache.carbondata.core.reader.CarbonIndexFileReader in project carbondata by apache.

the class CarbonIndexFileReaderTest method setUp.

@BeforeClass
public static void setUp() throws IOException {
    carbonIndexFileReader = new CarbonIndexFileReader();
    new MockUp<ThriftReader>() {

        @Mock
        public void open() throws IOException {
        }
    };
    carbonIndexFileReader.openThriftReader("TestFile.Carbon");
}