Examples with FileSplit org.apache.hadoop.mapred.FileSplit used on opensource projects

Example 66 with FileSplit

use of org.apache.hadoop.mapred.FileSplit in project ignite by apache.

the class HadoopV1Splitter method splitJob.

/**
 * @param jobConf Job configuration.
 * @return Collection of mapped splits.
 * @throws IgniteCheckedException If mapping failed.
 */
public static Collection<HadoopInputSplit> splitJob(JobConf jobConf) throws IgniteCheckedException {
    try {
        InputFormat<?, ?> format = jobConf.getInputFormat();
        assert format != null;
        InputSplit[] splits = format.getSplits(jobConf, 0);
        Collection<HadoopInputSplit> res = new ArrayList<>(splits.length);
        for (int i = 0; i < splits.length; i++) {
            InputSplit nativeSplit = splits[i];
            if (nativeSplit instanceof FileSplit) {
                FileSplit s = (FileSplit) nativeSplit;
                res.add(new HadoopFileBlock(s.getLocations(), s.getPath().toUri(), s.getStart(), s.getLength()));
            } else
                res.add(HadoopUtils.wrapSplit(i, nativeSplit, nativeSplit.getLocations()));
        }
        return res;
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}