Examples with TaskEventTASucceeded org.apache.tez.dag.app.dag.event.TaskEventTASucceeded used on opensource projects

Example 1 with TaskEventTASucceeded

use of org.apache.tez.dag.app.dag.event.TaskEventTASucceeded in project tez by apache.

the class TestVertexImpl method testExceptionFromII_HandleInputInitializerEvent.

@Test(timeout = 5000)
public void testExceptionFromII_HandleInputInitializerEvent() throws Exception {
    useCustomInitializer = true;
    customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.HandleInputInitializerEvent);
    EventHandlingRootInputInitializer initializer = (EventHandlingRootInputInitializer) customInitializer;
    setupPreDagCreation();
    dagPlan = createDAGPlanWithRunningInitializer();
    setupPostDagCreation();
    VertexImplWithRunningInputInitializer v1 = (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
    VertexImplWithRunningInputInitializer v2 = (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
    initVertex(v1);
    startVertex(v1);
    Assert.assertEquals(VertexState.RUNNING, v1.getState());
    Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
    dispatcher.await();
    // Wait for the initializer to be invoked - which may be a separate thread.
    while (!initializer.initStarted.get()) {
        Thread.sleep(10);
    }
    Assert.assertFalse(initializer.eventReceived.get());
    Assert.assertFalse(initializer.initComplete.get());
    // Signal the initializer by sending an event - via vertex1
    InputInitializerEvent event = InputInitializerEvent.create("vertex2", "input1", null);
    // Create taskId and taskAttemptId for the single task that exists in vertex1
    TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
    TezEvent tezEvent = new TezEvent(event, new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex2", ta0_t0_v1));
    // at least one task attempt is succeed, otherwise input initialize events won't been handled.
    dispatcher.getEventHandler().handle(new TaskEvent(t0_v1, TaskEventType.T_ATTEMPT_LAUNCHED));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(ta0_t0_v1));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
    dispatcher.await();
    // it would cause v2 fail as its II throw exception in handleInputInitializerEvent
    String diagnostics = StringUtils.join(v2.getDiagnostics(), ",");
    assertTrue(diagnostics.contains(IIExceptionLocation.HandleInputInitializerEvent.name()));
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v2.getTerminationCause());
}

Example 2 with TaskEventTASucceeded

use of org.apache.tez.dag.app.dag.event.TaskEventTASucceeded in project tez by apache.

the class TestVertexImpl method testFailuresMaxPercentSourceTaskAttemptCompletionEvents.

@Test(timeout = 5000)
public void testFailuresMaxPercentSourceTaskAttemptCompletionEvents() throws TezException {
    LOG.info("Testing testFailuresMaxPercentSourceTaskAttemptCompletionEvents");
    // Override the basic setup for this test to inject the specific config setting needed for this test
    useCustomInitializer = false;
    customInitializer = null;
    setupPreDagCreation();
    conf.setFloat(TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT, 50.0f);
    conf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    dagPlan = createTestDAGPlan();
    setupPostDagCreation();
    initAllVertices(VertexState.INITED);
    VertexImpl v4 = vertices.get("vertex4");
    VertexImpl v5 = vertices.get("vertex5");
    VertexImpl v6 = vertices.get("vertex6");
    startVertex(vertices.get("vertex1"));
    startVertex(vertices.get("vertex2"));
    dispatcher.await();
    LOG.info("Verifying v6 state " + v6.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    TezTaskID t1_v4 = TezTaskID.getInstance(v4.getVertexId(), 0);
    TezTaskID t2_v4 = TezTaskID.getInstance(v4.getVertexId(), 1);
    TezTaskID t1_v5 = TezTaskID.getInstance(v5.getVertexId(), 0);
    TezTaskID t2_v5 = TezTaskID.getInstance(v5.getVertexId(), 1);
    TezTaskAttemptID ta1_t1_v4 = TezTaskAttemptID.getInstance(t1_v4, 0);
    TezTaskAttemptID ta1_t2_v4 = TezTaskAttemptID.getInstance(t2_v4, 0);
    TezTaskAttemptID ta1_t1_v5 = TezTaskAttemptID.getInstance(t1_v5, 0);
    TezTaskAttemptID ta1_t2_v5 = TezTaskAttemptID.getInstance(t2_v5, 0);
    TaskSpec taskSpec = new TaskSpec("dag", "vertex", 2, new ProcessorDescriptor(), new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null, conf);
    TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(null, null);
    // Tasks can only succeed from a scheduled or running state
    dispatcher.getEventHandler().handle(new TaskEventScheduleTask(t1_v4, taskSpec, locationHint, false));
    dispatcher.getEventHandler().handle(new TaskEventScheduleTask(t2_v4, taskSpec, locationHint, false));
    // Completed tasks are less that the max percent failure
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t1_v4, TaskFailureType.NON_FATAL, null));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(ta1_t2_v4));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(ta1_t1_v5));
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t2_v5, TaskFailureType.NON_FATAL, null));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v4.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v5.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    Assert.assertEquals(4, v6.numSuccessSourceAttemptCompletions);
}

Example 3 with TaskEventTASucceeded

use of org.apache.tez.dag.app.dag.event.TaskEventTASucceeded in project tez by apache.

the class TestTaskImpl method testSucceededLeafTaskWithRetroFailures.

@SuppressWarnings("rawtypes")
@Test(timeout = 10000L)
public void testSucceededLeafTaskWithRetroFailures() throws InterruptedException {
    Configuration newConf = new Configuration(conf);
    newConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(newConf)).when(vertex).getVertexConfig();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, true, taskResource, containerContext, vertex);
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getAttemptList().get(0);
    launchTaskAttempt(firstMockTaskAttempt.getID());
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl secondMockTaskAttempt = mockTask.getAttemptList().get(1);
    launchTaskAttempt(secondMockTaskAttempt.getID());
    firstMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), 10, 10));
    secondMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), 10, 10));
    firstMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString())));
    firstMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString())));
    secondMockTaskAttempt.handle(new TaskAttemptEvent(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), TaskAttemptEventType.TA_DONE));
    firstMockTaskAttempt.handle(new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.CONTAINER_EXITED));
    mockTask.handle(new TaskEventTASucceeded(secondMockTaskAttempt.getID()));
    firstMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, firstMockTaskAttempt.getID(), "test", TaskAttemptTerminationCause.NO_PROGRESS));
    InputReadErrorEvent mockReEvent = InputReadErrorEvent.create("", 0, 0);
    TezTaskAttemptID mockDestId = firstMockTaskAttempt.getID();
    EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId);
    TezEvent tzEvent = new TezEvent(mockReEvent, meta);
    TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, tzEvent, 1);
    firstMockTaskAttempt.handle(outputFailedEvent);
    mockTask.handle(new TaskEventTAFailed(firstMockTaskAttempt.getID(), TaskFailureType.NON_FATAL, mock(TaskAttemptEvent.class)));
    Assert.assertEquals(mockTask.getInternalState(), TaskStateInternal.SUCCEEDED);
}

Example 4 with TaskEventTASucceeded

use of org.apache.tez.dag.app.dag.event.TaskEventTASucceeded in project tez by apache.

the class TestVertexImpl method testCompletedStatsCache.

@Test(timeout = 5000)
public void testCompletedStatsCache() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    dispatcher.getEventHandler().handle(new TaskEventTALaunched(TezTaskAttemptID.getInstance(t1, 0)));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(TezTaskAttemptID.getInstance(t1, 0)));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    VertexStatistics stats = v.getStatistics();
    // Ensure that task 0 is available in completed stats cache
    Assert.assertTrue(v.completedTasksStatsCache.taskSet.get(0));
    // Reschedule task 0
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    // cache should be cleared
    Assert.assertTrue(v.completedTasksStatsCache.taskSet.cardinality() == 0);
}

Example 5 with TaskEventTASucceeded

use of org.apache.tez.dag.app.dag.event.TaskEventTASucceeded in project tez by apache.

the class TestVertexImpl method testFailuresMaxPercentExceededSourceTaskAttemptCompletionEvents.

@Test(timeout = 5000)
public void testFailuresMaxPercentExceededSourceTaskAttemptCompletionEvents() throws TezException {
    LOG.info("Testing testFailuresMaxPercentSourceTaskAttemptCompletionEvents");
    // Override the basic setup for this test to inject the specific config setting needed for this test
    useCustomInitializer = false;
    customInitializer = null;
    setupPreDagCreation();
    conf.setFloat(TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT, 50.0f);
    conf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    dagPlan = createTestDAGPlan();
    setupPostDagCreation();
    initAllVertices(VertexState.INITED);
    VertexImpl v4 = vertices.get("vertex4");
    VertexImpl v5 = vertices.get("vertex5");
    VertexImpl v6 = vertices.get("vertex6");
    startVertex(vertices.get("vertex1"));
    startVertex(vertices.get("vertex2"));
    dispatcher.await();
    LOG.info("Verifying v6 state " + v6.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    TezTaskID t1_v4 = TezTaskID.getInstance(v4.getVertexId(), 0);
    TezTaskID t2_v4 = TezTaskID.getInstance(v4.getVertexId(), 1);
    TezTaskID t1_v5 = TezTaskID.getInstance(v5.getVertexId(), 0);
    TezTaskID t2_v5 = TezTaskID.getInstance(v5.getVertexId(), 1);
    TezTaskAttemptID ta1_t1_v4 = TezTaskAttemptID.getInstance(t1_v4, 0);
    TezTaskAttemptID ta1_t2_v4 = TezTaskAttemptID.getInstance(t2_v4, 0);
    TezTaskAttemptID ta1_t1_v5 = TezTaskAttemptID.getInstance(t1_v5, 0);
    TezTaskAttemptID ta1_t2_v5 = TezTaskAttemptID.getInstance(t2_v5, 0);
    TaskSpec taskSpec = new TaskSpec("dag", "vertex", 2, new ProcessorDescriptor(), new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null, conf);
    TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(null, null);
    // Tasks can only succeed from a scheduled or running state
    dispatcher.getEventHandler().handle(new TaskEventScheduleTask(t1_v4, taskSpec, locationHint, false));
    dispatcher.getEventHandler().handle(new TaskEventScheduleTask(t2_v4, taskSpec, locationHint, false));
    // Completed tasks are more that the max percent failure
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t1_v4, TaskFailureType.NON_FATAL, null));
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t2_v4, TaskFailureType.NON_FATAL, null));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(ta1_t1_v5));
    dispatcher.getEventHandler().handle(new TaskEventTAFailed(ta1_t2_v5, TaskFailureType.NON_FATAL, null));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v4.getState());
    Assert.assertEquals(VertexState.SUCCEEDED, v5.getState());
    Assert.assertEquals(VertexState.RUNNING, v6.getState());
    Assert.assertEquals(2, v6.numSuccessSourceAttemptCompletions);
}