Examples with SplitInfo com.tencent.angel.split.SplitInfo used on opensource projects

Example 1 with SplitInfo

use of com.tencent.angel.split.SplitInfo in project angel by Tencent.

the class ProtobufUtil method buildWorkerGroupMeta.

private static WorkerGroupMetaInfoProto buildWorkerGroupMeta(AMWorkerGroup group, SplitClassification splits, Configuration conf) throws IOException {
    WorkerGroupMetaInfoProto.Builder builder = WorkerGroupMetaInfoProto.newBuilder();
    builder.setWorkerGroupId(convertToIdProto(group.getId()));
    builder.setLeaderId(convertToIdProto(group.getLeader()));
    for (AMWorker w : group.getWorkerSet()) {
        builder.addWorkers(buildWorkerMetaProto(w));
    }
    if (splits != null) {
        List<SplitInfo> splitInfoList = SerdeUtils.serilizeSplits(splits, conf);
        SplitInfoProto.Builder splitBuilder = SplitInfoProto.newBuilder();
        for (SplitInfo split : splitInfoList) {
            builder.addSplits(splitBuilder.setSplitClass(split.getSplitClass()).setSplit(ByteString.copyFrom(split.getSplit())).build());
        }
    }
    return builder.build();
}

Example 2 with SplitInfo

use of com.tencent.angel.split.SplitInfo in project angel by Tencent.

the class SerdeUtils method deSerilizeSplits.

public static SplitClassification deSerilizeSplits(List<SplitInfo> splitInfoList, Configuration conf) throws ClassNotFoundException, IOException {
    boolean isUseNewAPI = conf.getBoolean("mapred.mapper.new-api", false);
    if (isUseNewAPI) {
        List<org.apache.hadoop.mapreduce.InputSplit> splitList = new ArrayList<org.apache.hadoop.mapreduce.InputSplit>();
        for (SplitInfo splitInfo : splitInfoList) {
            splitList.add(deSerilizeNewSplit(splitInfo, conf));
        }
        SplitClassification splits = new SplitClassification(null, splitList, true);
        return splits;
    } else {
        List<org.apache.hadoop.mapred.InputSplit> splitList = new ArrayList<org.apache.hadoop.mapred.InputSplit>();
        for (SplitInfo splitInfo : splitInfoList) {
            splitList.add(deSerilizeOldSplit(splitInfo, conf));
        }
        SplitClassification splits = new SplitClassification(splitList, null, true);
        return splits;
    }
}

Example 3 with SplitInfo

use of com.tencent.angel.split.SplitInfo in project angel by Tencent.

the class SerdeUtils method serilizeSplit.

@SuppressWarnings({ "unchecked", "rawtypes" })
public static SplitInfo serilizeSplit(org.apache.hadoop.mapred.InputSplit split, Configuration conf) throws IOException {
    if (factory == null) {
        factory = new SerializationFactory(conf);
    }
    DataOutputBuffer out = new DataOutputBuffer(1024);
    try {
        Serializer serializer = factory.getSerializer(split.getClass());
        serializer.open(out);
        serializer.serialize(split);
        SplitInfo ret = new SplitInfo(split.getClass().getName(), out.getData());
        return ret;
    } finally {
        out.close();
    }
}

Example 4 with SplitInfo

use of com.tencent.angel.split.SplitInfo in project angel by Tencent.

the class SerdeUtils method serilizeSplit.

@SuppressWarnings({ "unchecked", "rawtypes" })
public static SplitInfo serilizeSplit(org.apache.hadoop.mapreduce.InputSplit split, Configuration conf) throws IOException {
    if (factory == null) {
        factory = new SerializationFactory(conf);
    }
    DataOutputBuffer out = new DataOutputBuffer(1024);
    try {
        Serializer serializer = factory.getSerializer(split.getClass());
        serializer.open(out);
        serializer.serialize(split);
        SplitInfo ret = new SplitInfo(split.getClass().getName(), out.getData());
        return ret;
    } finally {
        out.close();
    }
}