Examples with TaskAttemptEventOutputFailed org.apache.tez.dag.app.dag.event.TaskAttemptEventOutputFailed used on opensource projects

Example 1 with TaskAttemptEventOutputFailed

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventOutputFailed in project tez by apache.

the class Edge method sendTezEventToSourceTasks.

public void sendTezEventToSourceTasks(TezEvent tezEvent) throws AMUserCodeException {
    Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time");
    if (!bufferEvents.get()) {
        switch(tezEvent.getEventType()) {
            case INPUT_READ_ERROR_EVENT:
                InputReadErrorEvent event = (InputReadErrorEvent) tezEvent.getEvent();
                TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
                int destTaskIndex = destAttemptId.getTaskID().getId();
                int srcTaskIndex;
                int numConsumers;
                try {
                    if (onDemandRouting) {
                        srcTaskIndex = ((EdgeManagerPluginOnDemand) edgeManager).routeInputErrorEventToSource(destTaskIndex, event.getIndex());
                    } else {
                        srcTaskIndex = edgeManager.routeInputErrorEventToSource(event, destTaskIndex, event.getIndex());
                    }
                    Preconditions.checkArgument(srcTaskIndex >= 0, "SourceTaskIndex should not be negative," + "srcTaskIndex=" + srcTaskIndex);
                    numConsumers = edgeManager.getNumDestinationConsumerTasks(srcTaskIndex);
                    Preconditions.checkArgument(numConsumers > 0, "ConsumerTaskNum must be positive," + "numConsumers=" + numConsumers);
                } catch (Exception e) {
                    throw new AMUserCodeException(Source.EdgeManager, "Fail to sendTezEventToSourceTasks, " + "TezEvent:" + tezEvent.getEvent() + "sourceInfo:" + tezEvent.getSourceInfo() + "destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
                }
                Task srcTask = sourceVertex.getTask(srcTaskIndex);
                if (srcTask == null) {
                    throw new TezUncheckedException("Unexpected null task." + " sourceVertex=" + sourceVertex.getLogIdentifier() + " srcIndex = " + srcTaskIndex + " destAttemptId=" + destAttemptId + " destIndex=" + destTaskIndex + " edgeManager=" + edgeManager.getClass().getName());
                }
                TezTaskID srcTaskId = srcTask.getTaskId();
                int taskAttemptIndex = event.getVersion();
                TezTaskAttemptID srcTaskAttemptId = TezTaskAttemptID.getInstance(srcTaskId, taskAttemptIndex);
                sendEvent(new TaskAttemptEventOutputFailed(srcTaskAttemptId, tezEvent, numConsumers));
                break;
            default:
                throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
        }
    } else {
        sourceEventBuffer.add(tezEvent);
    }
}

Example 2 with TaskAttemptEventOutputFailed

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventOutputFailed in project tez by apache.

the class TestTaskAttempt method testMultipleOutputFailed.

@Test(timeout = 5000)
public // TaskAttempt.
void testMultipleOutputFailed() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(1, 2);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
    TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
    MockEventHandler mockEh = new MockEventHandler();
    MockEventHandler eventHandler = spy(mockEh);
    TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
    Configuration taskConf = new Configuration();
    taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
    taskConf.setBoolean("fs.file.impl.disable.cache", true);
    locationHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null);
    Resource resource = Resource.newInstance(1024, 1);
    NodeId nid = NodeId.newInstance("127.0.0.1", 0);
    @SuppressWarnings("deprecation") ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(contId);
    when(container.getNodeId()).thenReturn(nid);
    when(container.getNodeHttpAddress()).thenReturn("localhost:0");
    AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appCtx);
    containers.addContainerIfNew(container, 0, 0, 0);
    doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
    doReturn(containers).when(appCtx).getAllContainers();
    HistoryEventHandler mockHistHandler = mock(HistoryEventHandler.class);
    doReturn(mockHistHandler).when(appCtx).getHistoryHandler();
    TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
    MockTaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mockHeartbeatHandler, appCtx, false, resource, createFakeContainerContext(), false);
    TezTaskAttemptID taskAttemptID = taImpl.getID();
    taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0));
    taImpl.handle(new TaskAttemptEventSubmitted(taskAttemptID, contId));
    taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID));
    verify(mockHeartbeatHandler).register(taskAttemptID);
    taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));
    assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED);
    verify(mockHeartbeatHandler).unregister(taskAttemptID);
    int expectedEventsTillSucceeded = 8;
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    ArgumentCaptor<DAGHistoryEvent> histArg = ArgumentCaptor.forClass(DAGHistoryEvent.class);
    verify(eventHandler, times(expectedEventsTillSucceeded)).handle(arg.capture());
    // start and finish
    verify(mockHistHandler, times(2)).handle(histArg.capture());
    DAGHistoryEvent histEvent = histArg.getValue();
    TaskAttemptFinishedEvent finishEvent = (TaskAttemptFinishedEvent) histEvent.getHistoryEvent();
    long finishTime = finishEvent.getFinishTime();
    verifyEventType(arg.getAllValues(), TaskEventTAUpdate.class, 2);
    InputReadErrorEvent mockReEvent = InputReadErrorEvent.create("", 0, 1);
    EventMetaData mockMeta = mock(EventMetaData.class);
    TezTaskAttemptID mockDestId1 = mock(TezTaskAttemptID.class);
    when(mockMeta.getTaskAttemptID()).thenReturn(mockDestId1);
    TezEvent tzEvent = new TezEvent(mockReEvent, mockMeta);
    taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 11));
    // failure threshold not met. state is SUCCEEDED
    assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED);
    // sending same error again doesnt change anything
    taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 11));
    assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED);
    // default value of error cause
    assertEquals(TaskAttemptTerminationCause.UNKNOWN_ERROR, taImpl.getTerminationCause());
    // different destination attempt reports error. now threshold crossed
    TezTaskAttemptID mockDestId2 = mock(TezTaskAttemptID.class);
    when(mockMeta.getTaskAttemptID()).thenReturn(mockDestId2);
    taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 11));
    assertEquals("Task attempt is not in FAILED state", taImpl.getState(), TaskAttemptState.FAILED);
    assertEquals(TaskAttemptTerminationCause.OUTPUT_LOST, taImpl.getTerminationCause());
    // verify unregister is not invoked again
    verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID);
    verify(mockHistHandler, times(3)).handle(histArg.capture());
    histEvent = histArg.getValue();
    finishEvent = (TaskAttemptFinishedEvent) histEvent.getHistoryEvent();
    assertEquals(TaskFailureType.NON_FATAL, finishEvent.getTaskFailureType());
    long newFinishTime = finishEvent.getFinishTime();
    Assert.assertEquals(finishTime, newFinishTime);
    assertEquals(true, taImpl.inputFailedReported);
    int expectedEventsAfterFetchFailure = expectedEventsTillSucceeded + 2;
    arg.getAllValues().clear();
    verify(eventHandler, times(expectedEventsAfterFetchFailure)).handle(arg.capture());
    Event e = verifyEventType(arg.getAllValues().subList(expectedEventsTillSucceeded, expectedEventsAfterFetchFailure), TaskEventTAFailed.class, 1);
    TaskEventTAFailed failedEvent = (TaskEventTAFailed) e;
    assertEquals(TaskFailureType.NON_FATAL, failedEvent.getTaskFailureType());
    taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 1));
    assertEquals("Task attempt is not in FAILED state, still", taImpl.getState(), TaskAttemptState.FAILED);
    assertFalse("InternalError occurred trying to handle TA_TOO_MANY_FETCH_FAILURES", eventHandler.internalError);
    // No new events.
    verify(eventHandler, times(expectedEventsAfterFetchFailure)).handle(arg.capture());
    taskConf.setInt(TezConfiguration.TEZ_TASK_MAX_ALLOWED_OUTPUT_FAILURES, 1);
    TezTaskID taskID2 = TezTaskID.getInstance(vertexID, 2);
    MockTaskAttemptImpl taImpl2 = new MockTaskAttemptImpl(taskID2, 1, eventHandler, taListener, taskConf, new SystemClock(), mockHeartbeatHandler, appCtx, false, resource, createFakeContainerContext(), false);
    TezTaskAttemptID taskAttemptID2 = taImpl2.getID();
    taImpl2.handle(new TaskAttemptEventSchedule(taskAttemptID2, 0, 0));
    taImpl2.handle(new TaskAttemptEventSubmitted(taskAttemptID2, contId));
    taImpl2.handle(new TaskAttemptEventStartedRemotely(taskAttemptID2));
    verify(mockHeartbeatHandler).register(taskAttemptID2);
    taImpl2.handle(new TaskAttemptEvent(taskAttemptID2, TaskAttemptEventType.TA_DONE));
    assertEquals("Task attempt is not in succeeded state", taImpl2.getState(), TaskAttemptState.SUCCEEDED);
    verify(mockHeartbeatHandler).unregister(taskAttemptID2);
    mockReEvent = InputReadErrorEvent.create("", 1, 1);
    mockMeta = mock(EventMetaData.class);
    mockDestId1 = mock(TezTaskAttemptID.class);
    when(mockMeta.getTaskAttemptID()).thenReturn(mockDestId1);
    tzEvent = new TezEvent(mockReEvent, mockMeta);
    // This should fail even when MAX_ALLOWED_OUTPUT_FAILURES_FRACTION is within limits, as
    // MAX_ALLOWED_OUTPUT_FAILURES has crossed the limit.
    taImpl2.handle(new TaskAttemptEventOutputFailed(taskAttemptID2, tzEvent, 8));
    assertEquals("Task attempt is not in failed state", taImpl2.getState(), TaskAttemptState.FAILED);
    assertEquals(TaskAttemptTerminationCause.OUTPUT_LOST, taImpl2.getTerminationCause());
    // verify unregister is not invoked again
    verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID2);
    Clock mockClock = mock(Clock.class);
    int readErrorTimespanSec = 1;
    taskConf.setInt(TezConfiguration.TEZ_TASK_MAX_ALLOWED_OUTPUT_FAILURES, 10);
    taskConf.setInt(TezConfiguration.TEZ_AM_MAX_ALLOWED_TIME_FOR_TASK_READ_ERROR_SEC, readErrorTimespanSec);
    TezTaskID taskID3 = TezTaskID.getInstance(vertexID, 3);
    MockTaskAttemptImpl taImpl3 = new MockTaskAttemptImpl(taskID3, 1, eventHandler, taListener, taskConf, mockClock, mockHeartbeatHandler, appCtx, false, resource, createFakeContainerContext(), false);
    TezTaskAttemptID taskAttemptID3 = taImpl3.getID();
    taImpl3.handle(new TaskAttemptEventSchedule(taskAttemptID3, 0, 0));
    taImpl3.handle(new TaskAttemptEventSubmitted(taskAttemptID3, contId));
    taImpl3.handle(new TaskAttemptEventStartedRemotely(taskAttemptID3));
    verify(mockHeartbeatHandler).register(taskAttemptID3);
    taImpl3.handle(new TaskAttemptEvent(taskAttemptID3, TaskAttemptEventType.TA_DONE));
    assertEquals("Task attempt is not in succeeded state", taImpl3.getState(), TaskAttemptState.SUCCEEDED);
    verify(mockHeartbeatHandler).unregister(taskAttemptID3);
    mockReEvent = InputReadErrorEvent.create("", 1, 1);
    mockMeta = mock(EventMetaData.class);
    mockDestId1 = mock(TezTaskAttemptID.class);
    when(mockMeta.getTaskAttemptID()).thenReturn(mockDestId1);
    tzEvent = new TezEvent(mockReEvent, mockMeta);
    when(mockClock.getTime()).thenReturn(1000L);
    // time deadline not exceeded for a couple of read error events
    taImpl3.handle(new TaskAttemptEventOutputFailed(taskAttemptID3, tzEvent, 1000));
    assertEquals("Task attempt is not in succeeded state", taImpl3.getState(), TaskAttemptState.SUCCEEDED);
    when(mockClock.getTime()).thenReturn(1500L);
    taImpl3.handle(new TaskAttemptEventOutputFailed(taskAttemptID3, tzEvent, 1000));
    assertEquals("Task attempt is not in succeeded state", taImpl3.getState(), TaskAttemptState.SUCCEEDED);
    // exceed the time threshold
    when(mockClock.getTime()).thenReturn(2001L);
    taImpl3.handle(new TaskAttemptEventOutputFailed(taskAttemptID3, tzEvent, 1000));
    assertEquals("Task attempt is not in FAILED state", taImpl3.getState(), TaskAttemptState.FAILED);
    assertEquals(TaskAttemptTerminationCause.OUTPUT_LOST, taImpl3.getTerminationCause());
    // verify unregister is not invoked again
    verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID3);
}

Example 3 with TaskAttemptEventOutputFailed

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventOutputFailed in project tez by apache.

the class TestTaskImpl method testTaskSucceedAndRetroActiveFailure.

@SuppressWarnings("rawtypes")
@Test(timeout = 5000)
public void testTaskSucceedAndRetroActiveFailure() {
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    launchTaskAttempt(mockTask.getLastAttempt().getID());
    updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.RUNNING);
    mockTask.handle(createTaskTASucceededEvent(mockTask.getLastAttempt().getID()));
    // The task should now have succeeded
    assertTaskSucceededState();
    verify(mockTask.stateChangeNotifier).taskSucceeded(any(String.class), eq(taskId), eq(mockTask.getLastAttempt().getID().getId()));
    ArgumentCaptor<DAGHistoryEvent> argumentCaptor = ArgumentCaptor.forClass(DAGHistoryEvent.class);
    verify(mockHistoryHandler).handle(argumentCaptor.capture());
    DAGHistoryEvent dagHistoryEvent = argumentCaptor.getValue();
    HistoryEvent historyEvent = dagHistoryEvent.getHistoryEvent();
    assertTrue(historyEvent instanceof TaskFinishedEvent);
    TaskFinishedEvent taskFinishedEvent = (TaskFinishedEvent) historyEvent;
    assertEquals(taskFinishedEvent.getFinishTime(), mockTask.getFinishTime());
    eventHandler.events.clear();
    // Now fail the attempt after it has succeeded
    TezTaskAttemptID mockDestId = mock(TezTaskAttemptID.class);
    TezEvent mockTezEvent = mock(TezEvent.class);
    EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId);
    when(mockTezEvent.getSourceInfo()).thenReturn(meta);
    TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, mockTezEvent, 1);
    mockTask.handle(createTaskTAFailedEvent(mockTask.getLastAttempt().getID(), TaskFailureType.NON_FATAL, outputFailedEvent));
    // The task should still be in the scheduled state
    assertTaskScheduledState();
    Event event = eventHandler.events.get(0);
    Assert.assertEquals(AMNodeEventType.N_TA_ENDED, event.getType());
    event = eventHandler.events.get(eventHandler.events.size() - 1);
    Assert.assertEquals(VertexEventType.V_TASK_RESCHEDULED, event.getType());
    // report of output read error should be the causal TA
    List<MockTaskAttemptImpl> attempts = mockTask.getAttemptList();
    Assert.assertEquals(2, attempts.size());
    MockTaskAttemptImpl newAttempt = attempts.get(1);
    Assert.assertEquals(mockDestId, newAttempt.getSchedulingCausalTA());
}

Example 4 with TaskAttemptEventOutputFailed

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventOutputFailed in project tez by apache.

the class TestTaskImpl method testSucceededLeafTaskWithRetroFailures.

@SuppressWarnings("rawtypes")
@Test(timeout = 10000L)
public void testSucceededLeafTaskWithRetroFailures() throws InterruptedException {
    Configuration newConf = new Configuration(conf);
    newConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(newConf)).when(vertex).getVertexConfig();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, true, taskResource, containerContext, vertex);
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getAttemptList().get(0);
    launchTaskAttempt(firstMockTaskAttempt.getID());
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl secondMockTaskAttempt = mockTask.getAttemptList().get(1);
    launchTaskAttempt(secondMockTaskAttempt.getID());
    firstMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), 10, 10));
    secondMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), 10, 10));
    firstMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString())));
    firstMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString())));
    secondMockTaskAttempt.handle(new TaskAttemptEvent(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), TaskAttemptEventType.TA_DONE));
    firstMockTaskAttempt.handle(new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.CONTAINER_EXITED));
    mockTask.handle(new TaskEventTASucceeded(secondMockTaskAttempt.getID()));
    firstMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, firstMockTaskAttempt.getID(), "test", TaskAttemptTerminationCause.NO_PROGRESS));
    InputReadErrorEvent mockReEvent = InputReadErrorEvent.create("", 0, 0);
    TezTaskAttemptID mockDestId = firstMockTaskAttempt.getID();
    EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId);
    TezEvent tzEvent = new TezEvent(mockReEvent, meta);
    TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, tzEvent, 1);
    firstMockTaskAttempt.handle(outputFailedEvent);
    mockTask.handle(new TaskEventTAFailed(firstMockTaskAttempt.getID(), TaskFailureType.NON_FATAL, mock(TaskAttemptEvent.class)));
    Assert.assertEquals(mockTask.getInternalState(), TaskStateInternal.SUCCEEDED);
}

Example 5 with TaskAttemptEventOutputFailed

use of org.apache.tez.dag.app.dag.event.TaskAttemptEventOutputFailed in project tez by apache.

the class TestTaskImpl method failAttempt.

private void failAttempt(MockTaskAttemptImpl taskAttempt, int index, int expectedIncompleteAttempts) {
    InputReadErrorEvent mockReEvent = InputReadErrorEvent.create("", 0, index);
    TezTaskAttemptID mockDestId = mock(TezTaskAttemptID.class);
    EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId);
    TezEvent tzEvent = new TezEvent(mockReEvent, meta);
    TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, tzEvent, 1);
    taskAttempt.handle(outputFailedEvent);
    TaskEvent tEventFail1 = new TaskEventTAFailed(taskAttempt.getID(), TaskFailureType.NON_FATAL, outputFailedEvent);
    mockTask.handle(tEventFail1);
    assertEquals("Unexpected number of incomplete attempts!", expectedIncompleteAttempts, mockTask.getUncompletedAttemptsCount());
}