Examples with SUCCESS_FILE_SUFFIX org.apache.carbondata.core.util.path.CarbonTablePath.SUCCESS_FILE_SUFFIX used on opensource projects

Example 1 with SUCCESS_FILE_SUFFIX

use of org.apache.carbondata.core.util.path.CarbonTablePath.SUCCESS_FILE_SUFFIX in project carbondata by apache.

the class StageInputCollector method collectStageFiles.

/**
 * Collect all stage files and matched success files.
 * A stage file without success file will not be collected
 */
public static void collectStageFiles(CarbonTable table, Configuration hadoopConf, List<CarbonFile> stageInputList, List<CarbonFile> successFileList) {
    Objects.requireNonNull(table);
    Objects.requireNonNull(hadoopConf);
    Objects.requireNonNull(stageInputList);
    Objects.requireNonNull(successFileList);
    CarbonFile dir = FileFactory.getCarbonFile(table.getStagePath(), hadoopConf);
    if (dir.exists()) {
        // list the stage folder and collect all stage files who has corresponding success file,
        // which means the file is committed
        CarbonFile[] allFiles = dir.listFiles();
        Map<String, CarbonFile> map = new HashMap<>();
        Arrays.stream(allFiles).filter(file -> file.getName().endsWith(SUCCESS_FILE_SUFFIX)).forEach(file -> map.put(file.getName().substring(0, file.getName().indexOf(".")), file));
        Arrays.stream(allFiles).filter(file -> !file.getName().endsWith(SUCCESS_FILE_SUFFIX)).filter(file -> map.containsKey(file.getName())).forEach(carbonFile -> {
            stageInputList.add(carbonFile);
            successFileList.add(map.get(carbonFile.getName()));
        });
    }
}