Examples with EdgeProperty org.apache.tez.dag.api.EdgeProperty used on opensource projects

Example 36 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestVertexImpl method testVertexConfiguredDoneByVMBeforeEdgeDefined.

@Test(timeout = 5000)
public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception {
    // Race when a source vertex manages to start before the target vertex has
    // been initialized
    setupPreDagCreation();
    dagPlan = createSamplerDAGPlan(true);
    setupPostDagCreation();
    VertexImpl vA = vertices.get("A");
    VertexImpl vB = vertices.get("B");
    VertexImpl vC = vertices.get("C");
    TestUpdateListener listener = new TestUpdateListener();
    updateTracker.registerForVertexUpdates(vB.getName(), EnumSet.of(org.apache.tez.dag.api.event.VertexState.CONFIGURED), listener);
    // fudge the vm so we can do custom stuff
    vB.vertexManager = new VertexManager(VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), vB, appContext, mock(StateChangeNotifier.class));
    vB.vertexReconfigurationPlanned();
    dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_INIT));
    dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITIALIZING, vA.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vB.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vC.getState());
    // setting the edge manager should vA to start
    EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    Edge e = vC.sourceVertices.get(vA);
    Assert.assertNull(e.getEdgeManager());
    e.setCustomEdgeManager(mockEdgeManagerDescriptor);
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, vA.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vB.getState());
    Assert.assertEquals(VertexState.INITIALIZING, vC.getState());
    // vB is not configured yet. Edge to C is not configured. So it should not send configured event
    // even thought VM says its doneConfiguring vertex
    vB.doneReconfiguringVertex();
    Assert.assertEquals(0, listener.events.size());
    // complete configuration and verify getting configured signal from vB
    EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"));
    Map<String, EdgeProperty> edges = Maps.newHashMap();
    edges.put("B", edgeProp);
    vC.reconfigureVertex(2, vertexLocationHint, edges);
    dispatcher.await();
    Assert.assertEquals(1, listener.events.size());
    Assert.assertEquals(vB.getName(), listener.events.get(0).getVertexName());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, listener.events.get(0).getVertexState());
    updateTracker.unregisterForVertexUpdates(vB.getName(), listener);
    Assert.assertEquals(VertexState.RUNNING, vA.getState());
    Assert.assertEquals(VertexState.RUNNING, vB.getState());
    Assert.assertEquals(VertexState.RUNNING, vC.getState());
    Assert.assertNotNull(vA.getTask(0));
    Assert.assertNotNull(vB.getTask(0));
    Assert.assertNotNull(vC.getTask(0));
}

Example 37 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestEdge method testCompositeEventHandling.

@SuppressWarnings({ "rawtypes" })
@Test(timeout = 5000)
public void testCompositeEventHandling() throws TezException {
    EventHandler eventHandler = mock(EventHandler.class);
    EdgeProperty edgeProp = EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class), mock(InputDescriptor.class));
    Edge edge = new Edge(edgeProp, eventHandler, new TezConfiguration());
    TezVertexID srcVertexID = createVertexID(1);
    TezVertexID destVertexID = createVertexID(2);
    LinkedHashMap<TezTaskID, Task> srcTasks = mockTasks(srcVertexID, 1);
    LinkedHashMap<TezTaskID, Task> destTasks = mockTasks(destVertexID, 5);
    TezTaskID srcTaskID = srcTasks.keySet().iterator().next();
    Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks);
    Vertex destVertex = mockVertex("dest", destVertexID, destTasks);
    edge.setSourceVertex(srcVertex);
    edge.setDestinationVertex(destVertex);
    edge.initialize();
    // Task0, Attempt 0
    TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2);
    EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
    // Verification via a CompositeEvent
    CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(), ByteBuffer.wrap("bytes".getBytes()));
    // AttemptNum
    cdmEvent.setVersion(2);
    TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
    // Event setup to look like it would after the Vertex is done with it.
    edge.sendTezEventToDestinationTasks(tezEvent);
    verifyEvents(srcTAID, destTasks);
    // Same Verification via regular DataMovementEvents
    // Reset the mock
    resetTaskMocks(destTasks.values());
    for (int i = 0; i < destTasks.size(); i++) {
        DataMovementEvent dmEvent = DataMovementEvent.create(i, ByteBuffer.wrap("bytes".getBytes()));
        dmEvent.setVersion(2);
        tezEvent = new TezEvent(dmEvent, srcMeta);
        edge.sendTezEventToDestinationTasks(tezEvent);
    }
    verifyEvents(srcTAID, destTasks);
}

Example 38 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class TestHistoryEventJsonConversion method testConvertVertexReconfigureDoneEvent.

@Test(timeout = 5000)
public void testConvertVertexReconfigureDoneEvent() throws JSONException {
    TezVertexID vId = TezVertexID.getInstance(TezDAGID.getInstance(ApplicationId.newInstance(1l, 1), 1), 1);
    Map<String, EdgeProperty> edgeMgrs = new HashMap<String, EdgeProperty>();
    edgeMgrs.put("a", EdgeProperty.create(EdgeManagerPluginDescriptor.create("a.class").setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")));
    VertexConfigurationDoneEvent event = new VertexConfigurationDoneEvent(vId, 0L, 1, null, edgeMgrs, null, true);
    JSONObject jsonObject = HistoryEventJsonConversion.convertToJson(event);
    Assert.assertNotNull(jsonObject);
    Assert.assertEquals(vId.toString(), jsonObject.getString(ATSConstants.ENTITY));
    Assert.assertEquals(ATSConstants.TEZ_VERTEX_ID, jsonObject.get(ATSConstants.ENTITY_TYPE));
    JSONArray events = jsonObject.getJSONArray(ATSConstants.EVENTS);
    Assert.assertEquals(1, events.length());
    JSONObject evt = events.getJSONObject(0);
    Assert.assertEquals(HistoryEventType.VERTEX_CONFIGURE_DONE.name(), evt.getString(ATSConstants.EVENT_TYPE));
    JSONObject evtInfo = evt.getJSONObject(ATSConstants.EVENT_INFO);
    Assert.assertEquals(1, evtInfo.getInt(ATSConstants.NUM_TASKS));
    Assert.assertNotNull(evtInfo.getJSONObject(ATSConstants.UPDATED_EDGE_MANAGERS));
    JSONObject updatedEdgeMgrs = evtInfo.getJSONObject(ATSConstants.UPDATED_EDGE_MANAGERS);
    Assert.assertEquals(1, updatedEdgeMgrs.length());
    Assert.assertNotNull(updatedEdgeMgrs.getJSONObject("a"));
    JSONObject updatedEdgeMgr = updatedEdgeMgrs.getJSONObject("a");
    Assert.assertEquals(DataMovementType.CUSTOM.name(), updatedEdgeMgr.getString(DAGUtils.DATA_MOVEMENT_TYPE_KEY));
    Assert.assertEquals("In", updatedEdgeMgr.getString(DAGUtils.EDGE_DESTINATION_CLASS_KEY));
    Assert.assertEquals("a.class", updatedEdgeMgr.getString(DAGUtils.EDGE_MANAGER_CLASS_KEY));
}

Example 39 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class CartesianProduct method createDAG.

private DAG createDAG(TezConfiguration tezConf, String inputPath1, String inputPath2, String inputPath3, String outputPath, boolean isPartitioned) throws IOException {
    Vertex v1 = Vertex.create(VERTEX1, ProcessorDescriptor.create(TokenProcessor.class.getName()));
    // turn off groupSplit so that each input file incurs one task
    v1.addDataSource(INPUT, MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath1).groupSplits(false).build());
    Vertex v2 = Vertex.create(VERTEX2, ProcessorDescriptor.create(TokenProcessor.class.getName()));
    v2.addDataSource(INPUT, MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath2).groupSplits(false).build());
    Vertex v3 = Vertex.create(VERTEX3, ProcessorDescriptor.create(TokenProcessor.class.getName()));
    v3.addDataSource(INPUT, MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath3).groupSplits(false).build());
    CartesianProductConfig cartesianProductConfig;
    if (isPartitioned) {
        Map<String, Integer> vertexPartitionMap = new HashMap<>();
        for (String vertex : cpSources) {
            vertexPartitionMap.put(vertex, numPartition);
        }
        cartesianProductConfig = new CartesianProductConfig(vertexPartitionMap);
    } else {
        cartesianProductConfig = new CartesianProductConfig(Arrays.asList(cpSources));
    }
    UserPayload userPayload = cartesianProductConfig.toUserPayload(tezConf);
    Vertex v4 = Vertex.create(VERTEX4, ProcessorDescriptor.create(JoinProcessor.class.getName()));
    v4.addDataSink(OUTPUT, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, outputPath).build());
    v4.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()).setUserPayload(userPayload));
    EdgeManagerPluginDescriptor cpEdgeManager = EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName());
    cpEdgeManager.setUserPayload(userPayload);
    EdgeProperty cpEdgeProperty;
    if (isPartitioned) {
        UnorderedPartitionedKVEdgeConfig cpEdgeConf = UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName(), CustomPartitioner.class.getName()).build();
        cpEdgeProperty = cpEdgeConf.createDefaultCustomEdgeProperty(cpEdgeManager);
    } else {
        UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName()).build();
        cpEdgeProperty = edgeConf.createDefaultCustomEdgeProperty(cpEdgeManager);
    }
    EdgeProperty broadcastEdgeProperty;
    UnorderedKVEdgeConfig broadcastEdgeConf = UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName()).build();
    broadcastEdgeProperty = broadcastEdgeConf.createDefaultBroadcastEdgeProperty();
    return DAG.create("CartesianProduct").addVertex(v1).addVertex(v2).addVertex(v3).addVertex(v4).addEdge(Edge.create(v1, v4, cpEdgeProperty)).addEdge(Edge.create(v2, v4, cpEdgeProperty)).addEdge(Edge.create(v3, v4, broadcastEdgeProperty));
}

Example 40 with EdgeProperty

use of org.apache.tez.dag.api.EdgeProperty in project tez by apache.

the class VertexConfigurationDoneEvent method fromProto.

public void fromProto(VertexConfigurationDoneProto proto) {
    this.vertexID = TezVertexID.fromString(proto.getVertexId());
    this.reconfigureDoneTime = proto.getReconfigureDoneTime();
    this.setParallelismCalledFlag = proto.getSetParallelismCalledFlag();
    this.numTasks = proto.getNumTasks();
    if (proto.hasVertexLocationHint()) {
        this.vertexLocationHint = DagTypeConverters.convertVertexLocationHintFromProto(proto.getVertexLocationHint());
    }
    if (proto.getEdgeManagerDescriptorsCount() > 0) {
        this.sourceEdgeProperties = new HashMap<String, EdgeProperty>(proto.getEdgeManagerDescriptorsCount());
        for (EdgeManagerDescriptorProto edgeManagerProto : proto.getEdgeManagerDescriptorsList()) {
            EdgeProperty edgeProperty = DagTypeConverters.convertFromProto(edgeManagerProto.getEdgeProperty());
            sourceEdgeProperties.put(edgeManagerProto.getEdgeName(), edgeProperty);
        }
    }
    if (proto.getRootInputSpecUpdatesCount() > 0) {
        this.rootInputSpecUpdates = Maps.newHashMap();
        for (RootInputSpecUpdateProto rootInputSpecUpdateProto : proto.getRootInputSpecUpdatesList()) {
            InputSpecUpdate specUpdate;
            if (rootInputSpecUpdateProto.getForAllWorkUnits()) {
                specUpdate = InputSpecUpdate.createAllTaskInputSpecUpdate(rootInputSpecUpdateProto.getNumPhysicalInputs(0));
            } else {
                specUpdate = InputSpecUpdate.createPerTaskInputSpecUpdate(rootInputSpecUpdateProto.getNumPhysicalInputsList());
            }
            this.rootInputSpecUpdates.put(rootInputSpecUpdateProto.getInputName(), specUpdate);
        }
    }
}