Examples with VertexEventTaskCompleted org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted used on opensource projects

Example 26 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testSourceTaskAttemptCompletionEvents.

@Test(timeout = 5000)
public void testSourceTaskAttemptCompletionEvents() {
    LOG.info("Testing testSourceTaskAttemptCompletionEvents");
    initAllVertices(VertexState.INITED);
    VertexImpl v4 = vertices.get("vertex4");
    VertexImpl v5 = vertices.get("vertex5");
    VertexImpl v6 = vertices.get("vertex6");
    startVertex(vertices.get("vertex1"));
    startVertex(vertices.get("vertex2"));
    dispatcher.await();
    LOG.info("Verifying v6 state " + v6.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    TezTaskID t1_v4 = TezTaskID.getInstance(v4.getVertexId(), 0);
    TezTaskID t2_v4 = TezTaskID.getInstance(v4.getVertexId(), 1);
    TezTaskID t1_v5 = TezTaskID.getInstance(v5.getVertexId(), 0);
    TezTaskID t2_v5 = TezTaskID.getInstance(v5.getVertexId(), 1);
    TezTaskAttemptID ta1_t1_v4 = TezTaskAttemptID.getInstance(t1_v4, 0);
    TezTaskAttemptID ta2_t1_v4 = TezTaskAttemptID.getInstance(t1_v4, 0);
    TezTaskAttemptID ta1_t2_v4 = TezTaskAttemptID.getInstance(t2_v4, 0);
    TezTaskAttemptID ta1_t1_v5 = TezTaskAttemptID.getInstance(t1_v5, 0);
    TezTaskAttemptID ta1_t2_v5 = TezTaskAttemptID.getInstance(t2_v5, 0);
    TezTaskAttemptID ta2_t2_v5 = TezTaskAttemptID.getInstance(t2_v5, 0);
    v4.handle(new VertexEventTaskAttemptCompleted(ta1_t1_v4, TaskAttemptStateInternal.FAILED));
    v4.handle(new VertexEventTaskAttemptCompleted(ta2_t1_v4, TaskAttemptStateInternal.SUCCEEDED));
    v4.handle(new VertexEventTaskAttemptCompleted(ta1_t2_v4, TaskAttemptStateInternal.SUCCEEDED));
    v5.handle(new VertexEventTaskAttemptCompleted(ta1_t1_v5, TaskAttemptStateInternal.SUCCEEDED));
    v5.handle(new VertexEventTaskAttemptCompleted(ta1_t2_v5, TaskAttemptStateInternal.FAILED));
    v5.handle(new VertexEventTaskAttemptCompleted(ta2_t2_v5, TaskAttemptStateInternal.SUCCEEDED));
    v4.handle(new VertexEventTaskCompleted(t1_v4, TaskState.SUCCEEDED));
    v4.handle(new VertexEventTaskCompleted(t2_v4, TaskState.SUCCEEDED));
    v5.handle(new VertexEventTaskCompleted(t1_v5, TaskState.SUCCEEDED));
    v5.handle(new VertexEventTaskCompleted(t2_v5, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v4.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v5.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    Assert.assertEquals(4, v6.numSuccessSourceAttemptCompletions);
}

Example 27 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testRouteEvent_RecoveredEvent.

@Test(timeout = 5000)
public void testRouteEvent_RecoveredEvent() throws IOException {
    doReturn(historyEventHandler).when(appContext).getHistoryHandler();
    doReturn(true).when(appContext).isRecoveryEnabled();
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = (VertexImpl) vertices.get("vertex1");
    VertexImpl v2 = (VertexImpl) vertices.get("vertex2");
    VertexImpl v3 = (VertexImpl) vertices.get("vertex3");
    startVertex(v1);
    startVertex(v2);
    TezTaskID taskId = TezTaskID.getInstance(v1.getVertexId(), 0);
    v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
    DataMovementEvent dmEvent = DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0]));
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(taskId, 0);
    TezEvent tezEvent1 = new TezEvent(dmEvent, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", taId));
    v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Lists.newArrayList(tezEvent1)));
    dispatcher.await();
    assertTrue(v3.pendingTaskEvents.size() != 0);
    // ArgumentCaptor<DAGHistoryEvent> argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
    // verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
    // verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
    v3.scheduleTasks(Lists.newArrayList(ScheduleTaskRequest.create(0, null)));
    dispatcher.await();
    assertTrue(v3.pendingTaskEvents.size() == 0);
// recovery events is not only handled one time
// argCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
// verify(historyEventHandler, atLeast(1)).handle(argCaptor.capture());
// verifyHistoryEvents(argCaptor.getAllValues(), HistoryEventType.VERTEX_DATA_MOVEMENT_EVENTS_GENERATED, 1);
}

Example 28 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testCompletedStatsCache.

@Test(timeout = 5000)
public void testCompletedStatsCache() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    dispatcher.getEventHandler().handle(new TaskEventTALaunched(TezTaskAttemptID.getInstance(t1, 0)));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(TezTaskAttemptID.getInstance(t1, 0)));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    VertexStatistics stats = v.getStatistics();
    // Ensure that task 0 is available in completed stats cache
    Assert.assertTrue(v.completedTasksStatsCache.taskSet.get(0));
    // Reschedule task 0
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    // cache should be cleared
    Assert.assertTrue(v.completedTasksStatsCache.taskSet.cardinality() == 0);
}

Example 29 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testInputInitializerVertexStateUpdates.

@Test(timeout = 10000)
public void testInputInitializerVertexStateUpdates() throws Exception {
    // v2 running an Input initializer, which is subscribed to events on v1.
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null);
    // Using the EventHandlingRootInputInitializer since it keeps the initializer alive till signalled,
    // which is required to track events that it receives.
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    initializer.setNumVertexStateUpdateEvents(3);
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    initVertex(v1);
    startVertex(v1);
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    // Make v1 succeed
    for (TezTaskID taskId : v1.getTasks().keySet()) {
        v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED));
    }
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
    // wait for state update events are received, this is done in the state notifier thread
    initializer.waitForVertexStateUpdate();
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, initializer.stateUpdates.get(0).getVertexState());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING, initializer.stateUpdates.get(1).getVertexState());
    Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED, initializer.stateUpdates.get(2).getVertexState());
}

Example 30 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestVertexImpl method testVertexCommit.

@Test(timeout = 5000)
public void testVertexCommit() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex6");
    startVertex(vertices.get("vertex1"));
    startVertex(vertices.get("vertex2"));
    CountingOutputCommitter committer = (CountingOutputCommitter) v.getOutputCommitter("outputx");
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, committer.commitCounter);
    Assert.assertEquals(0, committer.abortCounter);
    Assert.assertEquals(1, committer.initCounter);
    Assert.assertEquals(1, committer.setupCounter);
}