Examples with S3CarbonFile org.apache.carbondata.core.datastore.filesystem.S3CarbonFile used on opensource projects

Example 1 with S3CarbonFile

use of org.apache.carbondata.core.datastore.filesystem.S3CarbonFile in project carbondata by apache.

the class BlockletIndexUtil method createCarbonDataFileBlockMetaInfoMapping.

/**
 * This method will create file name to block Meta Info Mapping. This method will reduce the
 * number of nameNode calls and using this method one namenode will fetch 1000 entries
 *
 * @param segmentFilePath
 * @return
 * @throws IOException
 */
public static Map<String, BlockMetaInfo> createCarbonDataFileBlockMetaInfoMapping(String segmentFilePath, Configuration configuration) throws IOException {
    Map<String, BlockMetaInfo> fileNameToMetaInfoMapping = new TreeMap();
    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentFilePath, configuration);
    if (carbonFile instanceof AbstractDFSCarbonFile && !(carbonFile instanceof S3CarbonFile)) {
        PathFilter pathFilter = new PathFilter() {

            @Override
            public boolean accept(Path path) {
                return CarbonTablePath.isCarbonDataFile(path.getName());
            }
        };
        CarbonFile[] carbonFiles = carbonFile.locationAwareListFiles(pathFilter);
        for (CarbonFile file : carbonFiles) {
            String[] location = file.getLocations();
            long len = file.getSize();
            BlockMetaInfo blockMetaInfo = new BlockMetaInfo(location, len);
            fileNameToMetaInfoMapping.put(file.getPath(), blockMetaInfo);
        }
    }
    return fileNameToMetaInfoMapping;
}