use of org.apache.tez.dag.records.TezDAGID in project tez by apache.
the class TestRecoveryParser method testRecoverableSummary_VertexInCommitting.
@Test(timeout = 5000)
public void testRecoverableSummary_VertexInCommitting() throws IOException {
ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
AppContext appContext = mock(AppContext.class);
when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1"));
when(appContext.getClock()).thenReturn(new SystemClock());
when(mockDAGImpl.getID()).thenReturn(dagID);
RecoveryService rService = new RecoveryService(appContext);
Configuration conf = new Configuration();
conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
rService.init(conf);
rService.start();
DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
// write a DAGSubmittedEvent first to initialize summaryStream
rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null)));
// It should be fine to skip other events, just for testing.
rService.handle(new DAGHistoryEvent(dagID, new VertexCommitStartedEvent(TezVertexID.getInstance(dagID, 0), 0L)));
rService.stop();
DAGRecoveryData dagData = parser.parseRecoveryData();
assertEquals(dagID, dagData.recoveredDagID);
assertTrue(dagData.nonRecoverable);
assertTrue(dagData.reason.contains("Vertex Commit was in progress"));
}
use of org.apache.tez.dag.records.TezDAGID in project tez by apache.
the class TestTaskCommunicatorManager method testReportFailureFromTaskCommunicator.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskCommunicator() throws TezException {
String dagName = DAG_NAME;
EventHandler eventHandler = mock(EventHandler.class);
AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
doReturn("testTaskCommunicator").when(appContext).getTaskCommunicatorName(0);
doReturn(eventHandler).when(appContext).getEventHandler();
DAG dag = mock(DAG.class);
TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
doReturn(dagName).when(dag).getName();
doReturn(dagId).when(dag).getID();
doReturn(dag).when(appContext).getCurrentDAG();
NamedEntityDescriptor<TaskCommunicatorDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>("testTaskCommunicator", TaskCommForFailureTest.class.getName());
List<NamedEntityDescriptor> list = new LinkedList<>();
list.add(namedEntityDescriptor);
TaskCommunicatorManager taskCommManager = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), list);
try {
taskCommManager.init(new Configuration());
taskCommManager.start();
taskCommManager.registerRunningContainer(mock(ContainerId.class), 0);
ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
verify(eventHandler, times(1)).handle(argumentCaptor.capture());
Event rawEvent = argumentCaptor.getValue();
assertTrue(rawEvent instanceof DAGEventTerminateDag);
DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
assertTrue(killEvent.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
reset(eventHandler);
taskCommManager.dagComplete(dag);
argumentCaptor = ArgumentCaptor.forClass(Event.class);
verify(eventHandler, times(1)).handle(argumentCaptor.capture());
rawEvent = argumentCaptor.getValue();
assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
assertTrue(event.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
} finally {
taskCommManager.stop();
}
}
use of org.apache.tez.dag.records.TezDAGID in project tez by apache.
the class TestRootInputInitializerManager method testEventBeforeSuccess.
// Simple testing. No events if task doesn't succeed.
// Also exercises path where two attempts are reported as successful via the stateChangeNotifier.
// Primarily a failure scenario, when a Task moves back to running from success
// Order event1, success1, event2, success2
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEventBeforeSuccess() throws Exception {
InputDescriptor id = mock(InputDescriptor.class);
InputInitializerDescriptor iid = mock(InputInitializerDescriptor.class);
RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> rootInput = new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>("InputName", id, iid);
InputInitializer initializer = mock(InputInitializer.class);
InputInitializerContext initializerContext = mock(InputInitializerContext.class);
Vertex vertex = mock(Vertex.class);
StateChangeNotifier stateChangeNotifier = mock(StateChangeNotifier.class);
AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
RootInputInitializerManager.InitializerWrapper initializerWrapper = new RootInputInitializerManager.InitializerWrapper(rootInput, initializer, initializerContext, vertex, stateChangeNotifier, appContext);
ApplicationId appId = ApplicationId.newInstance(1000, 1);
TezDAGID dagId = TezDAGID.getInstance(appId, 1);
TezVertexID srcVertexId = TezVertexID.getInstance(dagId, 2);
TezTaskID srcTaskId1 = TezTaskID.getInstance(srcVertexId, 3);
Vertex srcVertex = mock(Vertex.class);
Task srcTask1 = mock(Task.class);
doReturn(TaskState.RUNNING).when(srcTask1).getState();
doReturn(srcTask1).when(srcVertex).getTask(srcTaskId1.getId());
when(appContext.getCurrentDAG().getVertex(any(String.class))).thenReturn(srcVertex);
String srcVertexName = "srcVertexName";
List<TezEvent> eventList = Lists.newLinkedList();
// First Attempt send event
TezTaskAttemptID srcTaskAttemptId11 = TezTaskAttemptID.getInstance(srcTaskId1, 1);
EventMetaData sourceInfo11 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId11);
InputInitializerEvent e1 = InputInitializerEvent.create("fakeVertex", "fakeInput", null);
TezEvent te1 = new TezEvent(e1, sourceInfo11);
eventList.add(te1);
initializerWrapper.handleInputInitializerEvents(eventList);
verify(initializer, never()).handleInputInitializerEvent(any(List.class));
eventList.clear();
// First attempt, Task success notification
initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId11.getId());
ArgumentCaptor<List> argumentCaptor = ArgumentCaptor.forClass(List.class);
verify(initializer, times(1)).handleInputInitializerEvent(argumentCaptor.capture());
List<InputInitializerEvent> invokedEvents = argumentCaptor.getValue();
assertEquals(1, invokedEvents.size());
reset(initializer);
// 2nd attempt send event
TezTaskAttemptID srcTaskAttemptId12 = TezTaskAttemptID.getInstance(srcTaskId1, 2);
EventMetaData sourceInfo12 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId12);
InputInitializerEvent e2 = InputInitializerEvent.create("fakeVertex", "fakeInput", null);
TezEvent te2 = new TezEvent(e2, sourceInfo12);
eventList.add(te2);
initializerWrapper.handleInputInitializerEvents(eventList);
verify(initializer, never()).handleInputInitializerEvent(any(List.class));
eventList.clear();
reset(initializer);
// 2nd attempt succeeded
initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId12.getId());
verify(initializer, never()).handleInputInitializerEvent(argumentCaptor.capture());
}
use of org.apache.tez.dag.records.TezDAGID in project tez by apache.
the class TestTaskSchedulerManager method testReportFailureFromTaskScheduler.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskScheduler() {
String dagName = DAG_NAME;
Configuration conf = new TezConfiguration();
String taskSchedulerName = "testTaskScheduler";
String expIdentifier = "[0:" + taskSchedulerName + "]";
EventHandler eventHandler = mock(EventHandler.class);
AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
doReturn(taskSchedulerName).when(appContext).getTaskSchedulerName(0);
doReturn(eventHandler).when(appContext).getEventHandler();
doReturn(conf).when(appContext).getAMConf();
InetSocketAddress address = new InetSocketAddress("host", 55000);
DAGClientServer dagClientServer = mock(DAGClientServer.class);
doReturn(address).when(dagClientServer).getBindAddress();
DAG dag = mock(DAG.class);
TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
doReturn(dagName).when(dag).getName();
doReturn(dagId).when(dag).getID();
doReturn(dag).when(appContext).getCurrentDAG();
NamedEntityDescriptor<TaskSchedulerDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>(taskSchedulerName, TaskSchedulerForFailureTest.class.getName());
List<NamedEntityDescriptor> list = new LinkedList<>();
list.add(namedEntityDescriptor);
TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, dagClientServer, eventHandler, mock(ContainerSignatureMatcher.class), mock(WebUIService.class), list, false, new HadoopShimsLoader(appContext.getAMConf()).getHadoopShim()) {
@Override
TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
// Avoid wrapping in threads
return rawContext;
}
};
try {
taskSchedulerManager.init(new TezConfiguration());
taskSchedulerManager.start();
taskSchedulerManager.getTotalResources(0);
ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
verify(eventHandler, times(1)).handle(argumentCaptor.capture());
Event rawEvent = argumentCaptor.getValue();
assertTrue(rawEvent instanceof DAGEventTerminateDag);
DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
assertTrue(killEvent.getDiagnosticInfo().contains(expIdentifier));
reset(eventHandler);
taskSchedulerManager.getAvailableResources(0);
argumentCaptor = ArgumentCaptor.forClass(Event.class);
verify(eventHandler, times(1)).handle(argumentCaptor.capture());
rawEvent = argumentCaptor.getValue();
assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
assertTrue(event.getDiagnosticInfo().contains(expIdentifier));
} finally {
taskSchedulerManager.stop();
}
}
use of org.apache.tez.dag.records.TezDAGID in project tez by apache.
the class TestTaskAttempt method testFailure.
@Test(timeout = 5000)
public void testFailure() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 2);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
TezDAGID dagID = TezDAGID.getInstance(appId, 1);
TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
MockEventHandler eventHandler = spy(new MockEventHandler());
TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();
Configuration taskConf = new Configuration();
taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
taskConf.setBoolean("fs.file.impl.disable.cache", true);
taskConf.setBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, true);
locationHint = TaskLocationHint.createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null);
Resource resource = Resource.newInstance(1024, 1);
NodeId nid = NodeId.newInstance("127.0.0.1", 0);
@SuppressWarnings("deprecation") ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
Container container = mock(Container.class);
when(container.getId()).thenReturn(contId);
when(container.getNodeId()).thenReturn(nid);
when(container.getNodeHttpAddress()).thenReturn("localhost:0");
AMContainerMap containers = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appCtx);
containers.addContainerIfNew(container, 0, 0, 0);
doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
doReturn(containers).when(appCtx).getAllContainers();
TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mockHeartbeatHandler, appCtx, false, resource, createFakeContainerContext(), false);
TezTaskAttemptID taskAttemptID = taImpl.getID();
ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0));
taImpl.handle(new TaskAttemptEventSubmitted(taskAttemptID, contId));
taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID));
assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(), TaskAttemptState.RUNNING);
verify(mockHeartbeatHandler).register(taskAttemptID);
int expectedEventsAtRunning = 6;
verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());
verifyEventType(arg.getAllValues().subList(0, expectedEventsAtRunning), SpeculatorEventTaskAttemptStatusUpdate.class, 1);
taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false)));
taImpl.handle(new TaskAttemptEventAttemptFailed(taskAttemptID, TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "0", TaskAttemptTerminationCause.APPLICATION_ERROR));
assertEquals("Task attempt is not in the FAIL_IN_PROGRESS state", taImpl.getInternalState(), TaskAttemptStateInternal.FAIL_IN_PROGRESS);
verify(mockHeartbeatHandler).unregister(taskAttemptID);
assertEquals(1, taImpl.getDiagnostics().size());
assertEquals("0", taImpl.getDiagnostics().get(0));
assertEquals(TaskAttemptTerminationCause.APPLICATION_ERROR, taImpl.getTerminationCause());
assertEquals(TaskAttemptStateInternal.FAIL_IN_PROGRESS, taImpl.getInternalState());
taImpl.handle(new TaskAttemptEventTezEventUpdate(taImpl.getID(), Collections.EMPTY_LIST));
assertFalse("InternalError occurred trying to handle TA_TEZ_EVENT_UPDATE in FAIL_IN_PROGRESS state", eventHandler.internalError);
taImpl.handle(new TaskAttemptEventContainerTerminated(contId, taskAttemptID, "1", TaskAttemptTerminationCause.CONTAINER_EXITED));
// verify unregister is not invoked again
verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID);
assertEquals(2, taImpl.getDiagnostics().size());
assertEquals("1", taImpl.getDiagnostics().get(1));
// err cause does not change
assertEquals(TaskAttemptTerminationCause.APPLICATION_ERROR, taImpl.getTerminationCause());
int expectedEvenstAfterTerminating = expectedEventsAtRunning + 5;
arg = ArgumentCaptor.forClass(Event.class);
verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());
Event e = verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), TaskEventTAFailed.class, 1);
TaskEventTAFailed failedEvent = (TaskEventTAFailed) e;
assertEquals(TaskFailureType.NON_FATAL, failedEvent.getTaskFailureType());
verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);
verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), DAGEventCounterUpdate.class, 1);
verifyEventType(arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), SpeculatorEventTaskAttemptStatusUpdate.class, 2);
}
Aggregations