Examples with MergedBlockIndex org.apache.carbondata.format.MergedBlockIndex used on opensource projects

Example 1 with MergedBlockIndex

use of org.apache.carbondata.format.MergedBlockIndex in project carbondata by apache.

the class SegmentIndexFileStore method readMergeFile.

/**
 * Read carbonindexmerge file and update the map
 *
 * @param mergeFilePath
 * @throws IOException
 */
private void readMergeFile(String mergeFilePath) throws IOException {
    ThriftReader thriftReader = new ThriftReader(mergeFilePath);
    try {
        thriftReader.open();
        MergedBlockIndexHeader indexHeader = readMergeBlockIndexHeader(thriftReader);
        MergedBlockIndex mergedBlockIndex = readMergeBlockIndex(thriftReader);
        List<String> file_names = indexHeader.getFile_names();
        List<ByteBuffer> fileData = mergedBlockIndex.getFileData();
        CarbonFile mergeFile = FileFactory.getCarbonFile(mergeFilePath);
        assert (file_names.size() == fileData.size());
        for (int i = 0; i < file_names.size(); i++) {
            carbonIndexMap.put(file_names.get(i), fileData.get(i).array());
            carbonIndexMapWithFullPath.put(mergeFile.getParentFile().getAbsolutePath() + CarbonCommonConstants.FILE_SEPARATOR + file_names.get(i), fileData.get(i).array());
        }
    } finally {
        thriftReader.close();
    }
}

Example 2 with MergedBlockIndex

use of org.apache.carbondata.format.MergedBlockIndex in project carbondata by apache.

the class CarbonIndexFileMergeWriter method writeMergeIndexFile.

private String writeMergeIndexFile(List<String> indexFileNamesTobeAdded, String segmentPath, Map<String, byte[]> indexMap) throws IOException {
    MergedBlockIndexHeader indexHeader = new MergedBlockIndexHeader();
    MergedBlockIndex mergedBlockIndex = new MergedBlockIndex();
    List<String> fileNames = new ArrayList<>(indexMap.size());
    List<ByteBuffer> data = new ArrayList<>(indexMap.size());
    for (Map.Entry<String, byte[]> entry : indexMap.entrySet()) {
        if (indexFileNamesTobeAdded == null || indexFileNamesTobeAdded.contains(entry.getKey())) {
            fileNames.add(entry.getKey());
            data.add(ByteBuffer.wrap(entry.getValue()));
        }
    }
    if (fileNames.size() > 0) {
        String mergeIndexName = System.currentTimeMillis() + CarbonTablePath.MERGE_INDEX_FILE_EXT;
        openThriftWriter(segmentPath + "/" + mergeIndexName);
        indexHeader.setFile_names(fileNames);
        mergedBlockIndex.setFileData(data);
        writeMergedBlockIndexHeader(indexHeader);
        writeMergedBlockIndex(mergedBlockIndex);
        close();
        return mergeIndexName;
    }
    return null;
}