Search in sources :

Example 16 with TaskAttemptEventSchedule

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule in project tez by apache.

the class TestTaskImpl method testKilledTaskTransitionWithLaunchedAttempt.

@Test(timeout = 30000)
public void testKilledTaskTransitionWithLaunchedAttempt() throws InterruptedException {
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(firstMockTaskAttempt.getID());
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl secondMockTaskAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(secondMockTaskAttempt.getID());
    firstMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), 10, 10));
    secondMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), 10, 10));
    firstMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString())));
    firstMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString())));
    mockTask.handle(new TaskEventTermination(mockTask.getTaskId(), TaskAttemptTerminationCause.FRAMEWORK_ERROR, "test"));
    secondMockTaskAttempt.handle(new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), "test", TaskAttemptTerminationCause.FRAMEWORK_ERROR));
    mockTask.handle(new TaskEventTAKilled(secondMockTaskAttempt.getID(), new TaskAttemptEvent(secondMockTaskAttempt.getID(), TaskAttemptEventType.TA_KILLED)));
    firstMockTaskAttempt.handle(new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), "test", TaskAttemptTerminationCause.FRAMEWORK_ERROR));
    mockTask.handle(new TaskEventTAKilled(firstMockTaskAttempt.getID(), new TaskAttemptEvent(firstMockTaskAttempt.getID(), TaskAttemptEventType.TA_KILLED)));
    firstMockTaskAttempt.handle(new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), "test", TaskAttemptTerminationCause.FRAMEWORK_ERROR));
    assertEquals("Task should have been killed!", mockTask.getInternalState(), TaskStateInternal.KILLED);
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl thirdMockTaskAttempt = mockTask.getLastAttempt();
    mockTask.handle(createTaskTALauncherEvent(thirdMockTaskAttempt.getID()));
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl fourthMockTaskAttempt = mockTask.getLastAttempt();
    mockTask.handle(createTaskTASucceededEvent(fourthMockTaskAttempt.getID()));
    MockTaskAttemptImpl fifthMockTaskAttempt = mockTask.getLastAttempt();
    mockTask.handle(createTaskTAFailedEvent(fifthMockTaskAttempt.getID()));
}
Also used : TaskAttemptEventAttemptKilled(org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptKilled) TaskAttemptEvent(org.apache.tez.dag.app.dag.event.TaskAttemptEvent) TaskAttemptEventSchedule(org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule) TaskAttemptEventSubmitted(org.apache.tez.dag.app.dag.event.TaskAttemptEventSubmitted) TezTaskID(org.apache.tez.dag.records.TezTaskID) TaskEventTAKilled(org.apache.tez.dag.app.dag.event.TaskEventTAKilled) TaskAttemptEventStartedRemotely(org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely) TaskEventTermination(org.apache.tez.dag.app.dag.event.TaskEventTermination) Test(org.junit.Test)

Example 17 with TaskAttemptEventSchedule

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule in project tez by apache.

the class TestTaskImpl method testSucceededLeafTaskWithRetroFailures.

@SuppressWarnings("rawtypes")
@Test(timeout = 10000L)
public void testSucceededLeafTaskWithRetroFailures() throws InterruptedException {
    Configuration newConf = new Configuration(conf);
    newConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(newConf)).when(vertex).getVertexConfig();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, true, taskResource, containerContext, vertex);
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getAttemptList().get(0);
    launchTaskAttempt(firstMockTaskAttempt.getID());
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl secondMockTaskAttempt = mockTask.getAttemptList().get(1);
    launchTaskAttempt(secondMockTaskAttempt.getID());
    firstMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), 10, 10));
    secondMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), 10, 10));
    firstMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString())));
    firstMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString())));
    secondMockTaskAttempt.handle(new TaskAttemptEvent(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), TaskAttemptEventType.TA_DONE));
    firstMockTaskAttempt.handle(new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.CONTAINER_EXITED));
    mockTask.handle(new TaskEventTASucceeded(secondMockTaskAttempt.getID()));
    firstMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, firstMockTaskAttempt.getID(), "test", TaskAttemptTerminationCause.NO_PROGRESS));
    InputReadErrorEvent mockReEvent = InputReadErrorEvent.create("", 0, 0);
    TezTaskAttemptID mockDestId = firstMockTaskAttempt.getID();
    EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId);
    TezEvent tzEvent = new TezEvent(mockReEvent, meta);
    TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, tzEvent, 1);
    firstMockTaskAttempt.handle(outputFailedEvent);
    mockTask.handle(new TaskEventTAFailed(firstMockTaskAttempt.getID(), TaskFailureType.NON_FATAL, mock(TaskAttemptEvent.class)));
    Assert.assertEquals(mockTask.getInternalState(), TaskStateInternal.SUCCEEDED);
}
Also used : Vertex(org.apache.tez.dag.app.dag.Vertex) Configuration(org.apache.hadoop.conf.Configuration) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) TaskEventTAFailed(org.apache.tez.dag.app.dag.event.TaskEventTAFailed) TaskAttemptEvent(org.apache.tez.dag.app.dag.event.TaskAttemptEvent) InputReadErrorEvent(org.apache.tez.runtime.api.events.InputReadErrorEvent) TaskAttemptEventSubmitted(org.apache.tez.dag.app.dag.event.TaskAttemptEventSubmitted) TaskEventTASucceeded(org.apache.tez.dag.app.dag.event.TaskEventTASucceeded) TezTaskID(org.apache.tez.dag.records.TezTaskID) TaskAttemptEventOutputFailed(org.apache.tez.dag.app.dag.event.TaskAttemptEventOutputFailed) TaskAttemptEventContainerTerminated(org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated) TaskAttemptEventSchedule(org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule) TezEvent(org.apache.tez.runtime.api.impl.TezEvent) EventMetaData(org.apache.tez.runtime.api.impl.EventMetaData) TaskAttemptEventStartedRemotely(org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely) TaskAttemptEventAttemptFailed(org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed) TezTaskAttemptID(org.apache.tez.dag.records.TezTaskAttemptID) Test(org.junit.Test)

Example 18 with TaskAttemptEventSchedule

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule in project tez by apache.

the class TestTaskImpl method testFailedTaskTransitionWithLaunchedAttempt.

@Test(timeout = 30000)
public void testFailedTaskTransitionWithLaunchedAttempt() throws InterruptedException {
    Configuration newConf = new Configuration(conf);
    newConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(newConf)).when(vertex).getVertexConfig();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, leafVertex, taskResource, containerContext, vertex);
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(firstMockTaskAttempt.getID());
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl secondMockTaskAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(secondMockTaskAttempt.getID());
    firstMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), 10, 10));
    secondMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), 10, 10));
    firstMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString())));
    firstMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString())));
    secondMockTaskAttempt.handle(new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.NO_PROGRESS));
    firstMockTaskAttempt.handle(new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.NO_PROGRESS));
    firstMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, firstMockTaskAttempt.getID(), "test", TaskAttemptTerminationCause.NO_PROGRESS));
    secondMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, secondMockTaskAttempt.getID(), "test", TaskAttemptTerminationCause.NO_PROGRESS));
    mockTask.handle(new TaskEventTAFailed(secondMockTaskAttempt.getID(), TaskFailureType.NON_FATAL, mock(TaskAttemptEvent.class)));
    mockTask.handle(new TaskEventTAFailed(firstMockTaskAttempt.getID(), TaskFailureType.NON_FATAL, mock(TaskAttemptEvent.class)));
    assertTrue("Attempts should have failed!", firstMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.FAILED && secondMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.FAILED);
    assertEquals("Task should have no uncompleted attempts!", 0, mockTask.getUncompletedAttemptsCount());
    assertTrue("Task should have failed!", mockTask.getState() == TaskState.FAILED);
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl thirdMockTaskAttempt = mockTask.getLastAttempt();
    mockTask.handle(createTaskTALauncherEvent(thirdMockTaskAttempt.getID()));
}
Also used : Vertex(org.apache.tez.dag.app.dag.Vertex) Configuration(org.apache.hadoop.conf.Configuration) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) TaskEventTAFailed(org.apache.tez.dag.app.dag.event.TaskEventTAFailed) TaskAttemptEventSubmitted(org.apache.tez.dag.app.dag.event.TaskAttemptEventSubmitted) TezTaskID(org.apache.tez.dag.records.TezTaskID) TaskAttemptEventContainerTerminated(org.apache.tez.dag.app.dag.event.TaskAttemptEventContainerTerminated) TaskAttemptEventSchedule(org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule) TaskAttemptEventStartedRemotely(org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely) TaskAttemptEventAttemptFailed(org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed) Test(org.junit.Test)

Example 19 with TaskAttemptEventSchedule

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule in project tez by apache.

the class TestVertexImpl method testPickupDagLocalResourceOnScheduleTask.

@Test(timeout = 5000)
public void testPickupDagLocalResourceOnScheduleTask() {
    initAllVertices(VertexState.INITED);
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(v1);
    TezTaskAttemptID taskAttemptId0 = TezTaskAttemptID.getInstance(v1.getTask(0).getTaskId(), 0);
    TaskAttemptImpl ta0 = (TaskAttemptImpl) v1.getTask(0).getAttempt(taskAttemptId0);
    ta0.handle(new TaskAttemptEventSchedule(taskAttemptId0, 1, 1));
    dispatcher.await();
    Assert.assertEquals(1, amSchedulerEventDispatcher.events.size());
    AMSchedulerEventTALaunchRequest launchRequestEvent = (AMSchedulerEventTALaunchRequest) amSchedulerEventDispatcher.events.get(0);
    Map<String, LocalResource> localResourceMap = launchRequestEvent.getContainerContext().getLocalResources();
    Assert.assertTrue(localResourceMap.containsKey("dag lr"));
    Assert.assertTrue(localResourceMap.containsKey("vertex lr"));
}
Also used : AMSchedulerEventTALaunchRequest(org.apache.tez.dag.app.rm.AMSchedulerEventTALaunchRequest) TaskAttemptEventSchedule(org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule) ByteString(com.google.protobuf.ByteString) TezTaskAttemptID(org.apache.tez.dag.records.TezTaskAttemptID) PlanLocalResource(org.apache.tez.dag.api.records.DAGProtos.PlanLocalResource) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource) VertexManagerPluginForTest(org.apache.tez.test.VertexManagerPluginForTest) Test(org.junit.Test) GraceShuffleVertexManagerForTest(org.apache.tez.test.GraceShuffleVertexManagerForTest) StateChangeNotifierForTest(org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest) EdgeManagerForTest(org.apache.tez.test.EdgeManagerForTest)

Example 20 with TaskAttemptEventSchedule

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule in project tez by apache.

the class TestVertexImpl method testVertexTaskAttemptProcessorFailure.

@SuppressWarnings("deprecation")
@Test(timeout = 5000)
public void testVertexTaskAttemptProcessorFailure() throws Exception {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex1");
    startVertex(v);
    dispatcher.await();
    TaskAttemptImpl ta = (TaskAttemptImpl) v.getTask(0).getAttempts().values().iterator().next();
    ta.handle(new TaskAttemptEventSchedule(ta.getID(), 2, 2));
    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
    ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(contId);
    when(container.getNodeId()).thenReturn(nid);
    when(container.getNodeHttpAddress()).thenReturn("localhost:0");
    AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
    containers.addContainerIfNew(container, 0, 0, 0);
    doReturn(containers).when(appContext).getAllContainers();
    ta.handle(new TaskAttemptEventSubmitted(ta.getID(), contId));
    ta.handle(new TaskAttemptEventStartedRemotely(ta.getID()));
    Assert.assertEquals(TaskAttemptStateInternal.RUNNING, ta.getInternalState());
    ta.handle(new TaskAttemptEventAttemptFailed(ta.getID(), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "diag", TaskAttemptTerminationCause.APPLICATION_ERROR));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    Assert.assertEquals(TaskAttemptTerminationCause.APPLICATION_ERROR, ta.getTerminationCause());
}
Also used : AMContainerMap(org.apache.tez.dag.app.rm.container.AMContainerMap) TaskCommunicatorManagerInterface(org.apache.tez.dag.app.TaskCommunicatorManagerInterface) ContainerContextMatcher(org.apache.tez.dag.app.rm.container.ContainerContextMatcher) TaskAttemptEventSubmitted(org.apache.tez.dag.app.dag.event.TaskAttemptEventSubmitted) Container(org.apache.hadoop.yarn.api.records.Container) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) NodeId(org.apache.hadoop.yarn.api.records.NodeId) TaskAttemptEventSchedule(org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule) ContainerHeartbeatHandler(org.apache.tez.dag.app.ContainerHeartbeatHandler) TaskAttemptEventStartedRemotely(org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely) TaskAttemptEventAttemptFailed(org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed) VertexManagerPluginForTest(org.apache.tez.test.VertexManagerPluginForTest) Test(org.junit.Test) GraceShuffleVertexManagerForTest(org.apache.tez.test.GraceShuffleVertexManagerForTest) StateChangeNotifierForTest(org.apache.tez.dag.app.dag.TestStateChangeNotifier.StateChangeNotifierForTest) EdgeManagerForTest(org.apache.tez.test.EdgeManagerForTest)

Aggregations

TaskAttemptEventSchedule (org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule)33 Test (org.junit.Test)31 TezTaskID (org.apache.tez.dag.records.TezTaskID)26 TaskCommunicatorManagerInterface (org.apache.tez.dag.app.TaskCommunicatorManagerInterface)24 TaskAttemptEventSubmitted (org.apache.tez.dag.app.dag.event.TaskAttemptEventSubmitted)24 Configuration (org.apache.hadoop.conf.Configuration)23 TezConfiguration (org.apache.tez.dag.api.TezConfiguration)23 TaskAttemptEventStartedRemotely (org.apache.tez.dag.app.dag.event.TaskAttemptEventStartedRemotely)23 TaskAttemptEvent (org.apache.tez.dag.app.dag.event.TaskAttemptEvent)21 Container (org.apache.hadoop.yarn.api.records.Container)20 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)20 NodeId (org.apache.hadoop.yarn.api.records.NodeId)20 SystemClock (org.apache.hadoop.yarn.util.SystemClock)20 TaskHeartbeatHandler (org.apache.tez.dag.app.TaskHeartbeatHandler)20 ContainerHeartbeatHandler (org.apache.tez.dag.app.ContainerHeartbeatHandler)19 AMContainerMap (org.apache.tez.dag.app.rm.container.AMContainerMap)19 ContainerContextMatcher (org.apache.tez.dag.app.rm.container.ContainerContextMatcher)19 TezTaskAttemptID (org.apache.tez.dag.records.TezTaskAttemptID)19 TaskStatusUpdateEvent (org.apache.tez.runtime.api.events.TaskStatusUpdateEvent)19 TezEvent (org.apache.tez.runtime.api.impl.TezEvent)19