Examples with SplitInfoProto com.tencent.angel.protobuf.generated.WorkerMasterServiceProtos.SplitInfoProto used on opensource projects

Example 1 with SplitInfoProto

use of com.tencent.angel.protobuf.generated.WorkerMasterServiceProtos.SplitInfoProto in project angel by Tencent.

the class SerdeUtils method deSerilizeSplitProtos.

public static SplitClassification deSerilizeSplitProtos(List<SplitInfoProto> splitInfoList, Configuration conf) throws ClassNotFoundException, IOException {
    boolean isUseNewAPI = conf.getBoolean("mapred.mapper.new-api", false);
    if (isUseNewAPI) {
        List<org.apache.hadoop.mapreduce.InputSplit> splitList = new ArrayList<org.apache.hadoop.mapreduce.InputSplit>();
        for (SplitInfoProto splitInfo : splitInfoList) {
            splitList.add(deSerilizeNewSplit(splitInfo.getSplitClass(), splitInfo.getSplit().toByteArray(), conf));
        }
        SplitClassification splits = new SplitClassification(null, splitList, true);
        return splits;
    } else {
        List<org.apache.hadoop.mapred.InputSplit> splitList = new ArrayList<org.apache.hadoop.mapred.InputSplit>();
        for (SplitInfoProto splitInfo : splitInfoList) {
            splitList.add(deSerilizeOldSplit(splitInfo.getSplitClass(), splitInfo.getSplit().toByteArray(), conf));
        }
        SplitClassification splits = new SplitClassification(splitList, null, true);
        return splits;
    }
}