Examples with DAGAppMasterEventUserServiceFatalError org.apache.tez.dag.app.dag.event.DAGAppMasterEventUserServiceFatalError used on opensource projects

Example 6 with DAGAppMasterEventUserServiceFatalError

use of org.apache.tez.dag.app.dag.event.DAGAppMasterEventUserServiceFatalError in project tez by apache.

the class TestTaskCommunicatorManager method testTaskCommunicatorUserError.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testTaskCommunicatorUserError() {
    TaskCommunicatorContextImpl taskCommContext = mock(TaskCommunicatorContextImpl.class);
    TaskCommunicator taskCommunicator = mock(TaskCommunicator.class, new ExceptionAnswer());
    doReturn(taskCommContext).when(taskCommunicator).getContext();
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(appContext.getEventHandler()).thenReturn(eventHandler);
    doReturn("testTaskCommunicator").when(appContext).getTaskCommunicatorName(0);
    String expectedId = "[0:testTaskCommunicator]";
    Configuration conf = new Configuration(false);
    TaskCommunicatorManager taskCommunicatorManager = new TaskCommunicatorManager(taskCommunicator, appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class));
    try {
        taskCommunicatorManager.init(conf);
        taskCommunicatorManager.start();
        // Invoking a couple of random methods.
        DAG mockDag = mock(DAG.class, RETURNS_DEEP_STUBS);
        when(mockDag.getID().getId()).thenReturn(1);
        taskCommunicatorManager.dagComplete(mockDag);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "dagComplete"));
        assertTrue(event.getDiagnosticInfo().contains("DAG completion"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
        when(appContext.getAllContainers().get(any(ContainerId.class)).getContainer().getNodeId()).thenReturn(mock(NodeId.class));
        taskCommunicatorManager.registerRunningContainer(mock(ContainerId.class), 0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(2)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getAllValues().get(1);
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "registerRunningContainer"));
        assertTrue(event.getDiagnosticInfo().contains("registering running Container"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
    } finally {
        taskCommunicatorManager.stop();
    }
}

Example 7 with DAGAppMasterEventUserServiceFatalError

use of org.apache.tez.dag.app.dag.event.DAGAppMasterEventUserServiceFatalError in project tez by apache.

the class TestTaskSchedulerManager method testTaskSchedulerUserError.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testTaskSchedulerUserError() {
    TaskScheduler taskScheduler = mock(TaskScheduler.class, new ExceptionAnswer());
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(appContext.getEventHandler()).thenReturn(eventHandler);
    doReturn("testTaskScheduler").when(appContext).getTaskSchedulerName(0);
    String expectedId = "[0:testTaskScheduler]";
    Configuration conf = new Configuration(false);
    InetSocketAddress address = new InetSocketAddress(15222);
    DAGClientServer mockClientService = mock(DAGClientServer.class);
    doReturn(address).when(mockClientService).getBindAddress();
    TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(taskScheduler, appContext, mock(ContainerSignatureMatcher.class), mockClientService, Executors.newFixedThreadPool(1)) {

        @Override
        protected void instantiateSchedulers(String host, int port, String trackingUrl, AppContext appContext) throws TezException {
        // Stubbed out since these are setup up front in the constructor used for testing
        }
    };
    try {
        taskSchedulerManager.init(conf);
        taskSchedulerManager.start();
        // Invoking a couple of random methods
        AMSchedulerEventTALaunchRequest launchRequest = new AMSchedulerEventTALaunchRequest(mock(TezTaskAttemptID.class), mock(Resource.class), mock(TaskSpec.class), mock(TaskAttempt.class), mock(TaskLocationHint.class), 0, mock(ContainerContext.class), 0, 0, 0);
        taskSchedulerManager.handleEvent(launchRequest);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "allocateTask"));
        assertTrue(event.getDiagnosticInfo().contains("Task Allocation"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
        taskSchedulerManager.dagCompleted();
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(2)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getAllValues().get(1);
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "dagComplete"));
        assertTrue(event.getDiagnosticInfo().contains("Dag Completion"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
    } finally {
        taskSchedulerManager.stop();
    }
}

Example 8 with DAGAppMasterEventUserServiceFatalError

use of org.apache.tez.dag.app.dag.event.DAGAppMasterEventUserServiceFatalError in project tez by apache.

the class TestTaskSchedulerManager method testReportFailureFromTaskScheduler.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskScheduler() {
    String dagName = DAG_NAME;
    Configuration conf = new TezConfiguration();
    String taskSchedulerName = "testTaskScheduler";
    String expIdentifier = "[0:" + taskSchedulerName + "]";
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn(taskSchedulerName).when(appContext).getTaskSchedulerName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(conf).when(appContext).getAMConf();
    InetSocketAddress address = new InetSocketAddress("host", 55000);
    DAGClientServer dagClientServer = mock(DAGClientServer.class);
    doReturn(address).when(dagClientServer).getBindAddress();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskSchedulerDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>(taskSchedulerName, TaskSchedulerForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, dagClientServer, eventHandler, mock(ContainerSignatureMatcher.class), mock(WebUIService.class), list, false, new HadoopShimsLoader(appContext.getAMConf()).getHadoopShim()) {

        @Override
        TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
            // Avoid wrapping in threads
            return rawContext;
        }
    };
    try {
        taskSchedulerManager.init(new TezConfiguration());
        taskSchedulerManager.start();
        taskSchedulerManager.getTotalResources(0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains(expIdentifier));
        reset(eventHandler);
        taskSchedulerManager.getAvailableResources(0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains(expIdentifier));
    } finally {
        taskSchedulerManager.stop();
    }
}

Example 9 with DAGAppMasterEventUserServiceFatalError

use of org.apache.tez.dag.app.dag.event.DAGAppMasterEventUserServiceFatalError in project tez by apache.

the class TestContainerLauncherManager method testContainerLauncherUserError.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testContainerLauncherUserError() throws ServicePluginException {
    ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn("testlauncher").when(appContext).getContainerLauncherName(0);
    Configuration conf = new Configuration(false);
    ContainerLauncherManager containerLauncherManager = new ContainerLauncherManager(appContext);
    containerLauncherManager.setContainerLauncher(containerLauncher);
    try {
        containerLauncherManager.init(conf);
        containerLauncherManager.start();
        // launch container
        doThrow(new RuntimeException("testexception")).when(containerLauncher).launchContainer(any(ContainerLaunchRequest.class));
        ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
        Container container1 = mock(Container.class);
        ContainerLauncherLaunchRequestEvent launchRequestEvent = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0);
        containerLauncherManager.handle(launchRequestEvent);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("testexception"));
        assertTrue(event.getDiagnosticInfo().contains("launching container"));
        assertTrue(event.getDiagnosticInfo().contains("[0:testlauncher]"));
        reset(eventHandler);
        // stop container
        doThrow(new RuntimeException("teststopexception")).when(containerLauncher).stopContainer(any(ContainerStopRequest.class));
        ContainerId containerId2 = mock(ContainerId.class);
        NodeId nodeId2 = mock(NodeId.class);
        ContainerLauncherStopRequestEvent stopRequestEvent = new ContainerLauncherStopRequestEvent(containerId2, nodeId2, null, 0, 0, 0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        containerLauncherManager.handle(stopRequestEvent);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertTrue(event.getError().getMessage().contains("teststopexception"));
        assertTrue(event.getDiagnosticInfo().contains("stopping container"));
        assertTrue(event.getDiagnosticInfo().contains("[0:testlauncher]"));
    } finally {
        containerLauncherManager.stop();
    }
}

Example 10 with DAGAppMasterEventUserServiceFatalError

use of org.apache.tez.dag.app.dag.event.DAGAppMasterEventUserServiceFatalError in project tez by apache.

the class TaskSchedulerManager method handleTaLaunchRequest.

private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
    TaskAttempt taskAttempt = event.getTaskAttempt();
    TaskLocationHint locationHint = event.getLocationHint();
    String[] hosts = null;
    String[] racks = null;
    if (locationHint != null) {
        TaskBasedLocationAffinity taskAffinity = locationHint.getAffinitizedTask();
        if (taskAffinity != null) {
            Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName());
            Preconditions.checkNotNull(vertex, "Invalid vertex in task based affinity " + taskAffinity + " for attempt: " + taskAttempt.getID());
            int taskIndex = taskAffinity.getTaskIndex();
            Preconditions.checkState(taskIndex >= 0 && taskIndex < vertex.getTotalTasks(), "Invalid taskIndex in task based affinity " + taskAffinity + " for attempt: " + taskAttempt.getID());
            TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
            if (affinityAttempt != null) {
                Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
                try {
                    taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt, event.getCapability(), affinityAttempt.getAssignedContainerID(), Priority.newInstance(event.getPriority()), event.getContainerContext(), event);
                } catch (Exception e) {
                    String msg = "Error in TaskScheduler for handling Task Allocation" + ", eventType=" + event.getType() + ", scheduler=" + Utils.getTaskSchedulerIdentifierString(event.getSchedulerId(), appContext) + ", taskAttemptId=" + taskAttempt.getID();
                    LOG.error(msg, e);
                    sendEvent(new DAGAppMasterEventUserServiceFatalError(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, msg, e));
                }
                return;
            }
            LOG.info("No attempt for task affinity to " + taskAffinity + " for attempt " + taskAttempt.getID() + " Ignoring.");
        // fall through with null hosts/racks
        } else {
            hosts = (locationHint.getHosts() != null) ? locationHint.getHosts().toArray(new String[locationHint.getHosts().size()]) : null;
            racks = (locationHint.getRacks() != null) ? locationHint.getRacks().toArray(new String[locationHint.getRacks().size()]) : null;
        }
    }
    try {
        taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt, event.getCapability(), hosts, racks, Priority.newInstance(event.getPriority()), event.getContainerContext(), event);
    } catch (Exception e) {
        String msg = "Error in TaskScheduler for handling Task Allocation" + ", eventType=" + event.getType() + ", scheduler=" + Utils.getTaskSchedulerIdentifierString(event.getSchedulerId(), appContext) + ", taskAttemptId=" + taskAttempt.getID();
        LOG.error(msg, e);
        sendEvent(new DAGAppMasterEventUserServiceFatalError(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, msg, e));
    }
}