Examples with SegmentIndexFileStore org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore used on opensource projects

Example 6 with SegmentIndexFileStore

use of org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore in project carbondata by apache.

the class CarbonIndexFileMergeWriter method writeMergeIndexFileBasedOnSegmentFile.

private String writeMergeIndexFileBasedOnSegmentFile(String segmentId, List<String> indexFileNamesTobeAdded, SegmentFileStore sfs, CarbonFile[] indexFiles) throws IOException {
    SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
    fileStore.readAllIIndexOfSegment(sfs.getSegmentFile(), sfs.getTablePath(), SegmentStatus.SUCCESS, true);
    Map<String, byte[]> indexMap = fileStore.getCarbonIndexMapWithFullPath();
    Map<String, Map<String, byte[]>> indexLocationMap = new HashMap<>();
    for (Map.Entry<String, byte[]> entry : indexMap.entrySet()) {
        Path path = new Path(entry.getKey());
        Map<String, byte[]> map = indexLocationMap.get(path.getParent().toString());
        if (map == null) {
            map = new HashMap<>();
            indexLocationMap.put(path.getParent().toString(), map);
        }
        map.put(path.getName(), entry.getValue());
    }
    for (Map.Entry<String, Map<String, byte[]>> entry : indexLocationMap.entrySet()) {
        String mergeIndexFile = writeMergeIndexFile(indexFileNamesTobeAdded, entry.getKey(), entry.getValue());
        for (Map.Entry<String, SegmentFileStore.FolderDetails> segentry : sfs.getLocationMap().entrySet()) {
            String location = segentry.getKey();
            if (segentry.getValue().isRelative()) {
                location = sfs.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + location;
            }
            if (new Path(entry.getKey()).equals(new Path(location))) {
                segentry.getValue().setMergeFileName(mergeIndexFile);
                break;
            }
        }
    }
    String uniqueId = String.valueOf(System.currentTimeMillis());
    String newSegmentFileName = SegmentFileStore.genSegmentFileName(segmentId, String.valueOf(uniqueId)) + CarbonTablePath.SEGMENT_EXT;
    String path = CarbonTablePath.getSegmentFilesLocation(table.getTablePath()) + CarbonCommonConstants.FILE_SEPARATOR + newSegmentFileName;
    SegmentFileStore.writeSegmentFile(sfs.getSegmentFile(), path);
    SegmentFileStore.updateSegmentFile(table.getTablePath(), segmentId, newSegmentFileName);
    for (CarbonFile file : indexFiles) {
        file.delete();
    }
    return uniqueId;
}

Example 7 with SegmentIndexFileStore

use of org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore in project carbondata by apache.

the class SegmentFileStore method getSegmentFileForPhysicalDataPartitions.

/**
 * It provides segment file only for the partitions which has physical index files.
 *
 * @param partitionSpecs
 */
public static SegmentFile getSegmentFileForPhysicalDataPartitions(String tablePath, List<PartitionSpec> partitionSpecs) throws IOException {
    SegmentFile segmentFile = null;
    for (PartitionSpec spec : partitionSpecs) {
        String location = spec.getLocation().toString();
        CarbonFile carbonFile = FileFactory.getCarbonFile(location);
        CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {

            @Override
            public boolean accept(CarbonFile file) {
                return CarbonTablePath.isCarbonIndexFile(file.getAbsolutePath());
            }
        });
        if (listFiles != null && listFiles.length > 0) {
            boolean isRelative = false;
            if (location.startsWith(tablePath)) {
                location = location.substring(tablePath.length(), location.length());
                isRelative = true;
            }
            SegmentFile localSegmentFile = new SegmentFile();
            FolderDetails folderDetails = new FolderDetails();
            folderDetails.setRelative(isRelative);
            folderDetails.setPartitions(spec.getPartitions());
            folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage());
            for (CarbonFile file : listFiles) {
                if (file.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
                    List<String> indexFiles = new SegmentIndexFileStore().getIndexFilesFromMergeFile(file.getAbsolutePath());
                    folderDetails.getFiles().addAll(indexFiles);
                    folderDetails.setMergeFileName(file.getName());
                } else {
                    folderDetails.getFiles().add(file.getName());
                }
            }
            localSegmentFile.addPath(location, folderDetails);
            if (segmentFile == null) {
                segmentFile = localSegmentFile;
            } else {
                segmentFile = segmentFile.merge(localSegmentFile);
            }
        }
    }
    return segmentFile;
}

Example 8 with SegmentIndexFileStore

use of org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore in project carbondata by apache.

the class BlockletDataMapIndexStore method get.

@Override
public BlockletDataMap get(TableBlockIndexUniqueIdentifier identifier) throws IOException {
    String lruCacheKey = identifier.getUniqueTableSegmentIdentifier();
    BlockletDataMap dataMap = (BlockletDataMap) lruCache.get(lruCacheKey);
    if (dataMap == null) {
        try {
            SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
            Set<String> filesRead = new HashSet<>();
            Map<String, BlockMetaInfo> blockMetaInfoMap = getBlockMetaInfoMap(identifier, indexFileStore, filesRead);
            dataMap = loadAndGetDataMap(identifier, indexFileStore, blockMetaInfoMap);
        } catch (MemoryException e) {
            LOGGER.error("memory exception when loading datamap: " + e.getMessage());
            throw new RuntimeException(e.getMessage(), e);
        }
    }
    return dataMap;
}