use of org.apache.tez.dag.api.records.DAGProtos.EdgePlan in project tez by apache.
the class DAGImpl method parseVertexEdges.
// hooks up this VertexImpl to input and output EdgeProperties
private static void parseVertexEdges(DAGImpl dag, Map<String, EdgePlan> edgePlans, Vertex vertex) {
VertexPlan vertexPlan = vertex.getVertexPlan();
Map<Vertex, Edge> inVertices = new HashMap<Vertex, Edge>();
Map<Vertex, Edge> outVertices = new HashMap<Vertex, Edge>();
for (String inEdgeId : vertexPlan.getInEdgeIdList()) {
EdgePlan edgePlan = edgePlans.get(inEdgeId);
Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());
Edge edge = dag.edges.get(inEdgeId);
edge.setSourceVertex(inVertex);
edge.setDestinationVertex(vertex);
inVertices.put(inVertex, edge);
}
for (String outEdgeId : vertexPlan.getOutEdgeIdList()) {
EdgePlan edgePlan = edgePlans.get(outEdgeId);
Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());
Edge edge = dag.edges.get(outEdgeId);
edge.setSourceVertex(vertex);
edge.setDestinationVertex(outVertex);
outVertices.put(outVertex, edge);
}
vertex.setInputVertices(inVertices);
vertex.setOutputVertices(outVertices);
}
use of org.apache.tez.dag.api.records.DAGProtos.EdgePlan in project tez by apache.
the class TestDAGPlan method testUserPayloadSerde.
@Test(timeout = 5000)
public void testUserPayloadSerde() {
DAG dag = DAG.create("testDag");
ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>()).addTaskLocalFiles(new HashMap<String, LocalResource>());
InputDescriptor inputDescriptor = InputDescriptor.create("input").setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
OutputDescriptor outputDescriptor = OutputDescriptor.create("output").setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
Edge edge = Edge.create(v1, v2, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
dag.addVertex(v1).addVertex(v2).addEdge(edge);
DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);
assertEquals(2, dagProto.getVertexCount());
assertEquals(1, dagProto.getEdgeCount());
VertexPlan v1Proto = dagProto.getVertex(0);
VertexPlan v2Proto = dagProto.getVertex(1);
EdgePlan edgeProto = dagProto.getEdge(0);
assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()));
assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());
assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray()));
assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());
assertEquals("inputBytes", new String(edgeProto.getEdgeDestination().getTezUserPayload().getUserPayload().toByteArray()));
assertEquals("input", edgeProto.getEdgeDestination().getClassName());
assertEquals("outputBytes", new String(edgeProto.getEdgeSource().getTezUserPayload().getUserPayload().toByteArray()));
assertEquals("output", edgeProto.getEdgeSource().getClassName());
EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));
byte[] ib = edgeProperty.getEdgeDestination().getUserPayload().deepCopyAsArray();
assertEquals("inputBytes", new String(ib));
assertEquals("input", edgeProperty.getEdgeDestination().getClassName());
byte[] ob = edgeProperty.getEdgeSource().getUserPayload().deepCopyAsArray();
assertEquals("outputBytes", new String(ob));
assertEquals("output", edgeProperty.getEdgeSource().getClassName());
}
Aggregations