Examples with EventHandler org.apache.hadoop.yarn.event.EventHandler used on opensource projects

Example 41 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestTaskCommunicatorManager method testReportFailureFromTaskCommunicator.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskCommunicator() throws TezException {
    String dagName = DAG_NAME;
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn("testTaskCommunicator").when(appContext).getTaskCommunicatorName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskCommunicatorDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>("testTaskCommunicator", TaskCommForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskCommunicatorManager taskCommManager = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), list);
    try {
        taskCommManager.init(new Configuration());
        taskCommManager.start();
        taskCommManager.registerRunningContainer(mock(ContainerId.class), 0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
        reset(eventHandler);
        taskCommManager.dagComplete(dag);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
    } finally {
        taskCommManager.stop();
    }
}

Example 42 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestTaskCommunicatorManager method testTaskCommunicatorUserError.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testTaskCommunicatorUserError() {
    TaskCommunicatorContextImpl taskCommContext = mock(TaskCommunicatorContextImpl.class);
    TaskCommunicator taskCommunicator = mock(TaskCommunicator.class, new ExceptionAnswer());
    doReturn(taskCommContext).when(taskCommunicator).getContext();
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(appContext.getEventHandler()).thenReturn(eventHandler);
    doReturn("testTaskCommunicator").when(appContext).getTaskCommunicatorName(0);
    String expectedId = "[0:testTaskCommunicator]";
    Configuration conf = new Configuration(false);
    TaskCommunicatorManager taskCommunicatorManager = new TaskCommunicatorManager(taskCommunicator, appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class));
    try {
        taskCommunicatorManager.init(conf);
        taskCommunicatorManager.start();
        // Invoking a couple of random methods.
        DAG mockDag = mock(DAG.class, RETURNS_DEEP_STUBS);
        when(mockDag.getID().getId()).thenReturn(1);
        taskCommunicatorManager.dagComplete(mockDag);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "dagComplete"));
        assertTrue(event.getDiagnosticInfo().contains("DAG completion"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
        when(appContext.getAllContainers().get(any(ContainerId.class)).getContainer().getNodeId()).thenReturn(mock(NodeId.class));
        taskCommunicatorManager.registerRunningContainer(mock(ContainerId.class), 0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(2)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getAllValues().get(1);
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "registerRunningContainer"));
        assertTrue(event.getDiagnosticInfo().contains("registering running Container"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
    } finally {
        taskCommunicatorManager.stop();
    }
}

Example 43 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestTaskSchedulerManager method testTaskSchedulerUserError.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testTaskSchedulerUserError() {
    TaskScheduler taskScheduler = mock(TaskScheduler.class, new ExceptionAnswer());
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(appContext.getEventHandler()).thenReturn(eventHandler);
    doReturn("testTaskScheduler").when(appContext).getTaskSchedulerName(0);
    String expectedId = "[0:testTaskScheduler]";
    Configuration conf = new Configuration(false);
    InetSocketAddress address = new InetSocketAddress(15222);
    DAGClientServer mockClientService = mock(DAGClientServer.class);
    doReturn(address).when(mockClientService).getBindAddress();
    TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(taskScheduler, appContext, mock(ContainerSignatureMatcher.class), mockClientService, Executors.newFixedThreadPool(1)) {

        @Override
        protected void instantiateSchedulers(String host, int port, String trackingUrl, AppContext appContext) throws TezException {
        // Stubbed out since these are setup up front in the constructor used for testing
        }
    };
    try {
        taskSchedulerManager.init(conf);
        taskSchedulerManager.start();
        // Invoking a couple of random methods
        AMSchedulerEventTALaunchRequest launchRequest = new AMSchedulerEventTALaunchRequest(mock(TezTaskAttemptID.class), mock(Resource.class), mock(TaskSpec.class), mock(TaskAttempt.class), mock(TaskLocationHint.class), 0, mock(ContainerContext.class), 0, 0, 0);
        taskSchedulerManager.handleEvent(launchRequest);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "allocateTask"));
        assertTrue(event.getDiagnosticInfo().contains("Task Allocation"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
        taskSchedulerManager.dagCompleted();
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(2)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getAllValues().get(1);
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "dagComplete"));
        assertTrue(event.getDiagnosticInfo().contains("Dag Completion"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
    } finally {
        taskSchedulerManager.stop();
    }
}

Example 44 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestTaskSchedulerManager method testReportFailureFromTaskScheduler.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskScheduler() {
    String dagName = DAG_NAME;
    Configuration conf = new TezConfiguration();
    String taskSchedulerName = "testTaskScheduler";
    String expIdentifier = "[0:" + taskSchedulerName + "]";
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn(taskSchedulerName).when(appContext).getTaskSchedulerName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(conf).when(appContext).getAMConf();
    InetSocketAddress address = new InetSocketAddress("host", 55000);
    DAGClientServer dagClientServer = mock(DAGClientServer.class);
    doReturn(address).when(dagClientServer).getBindAddress();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskSchedulerDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>(taskSchedulerName, TaskSchedulerForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, dagClientServer, eventHandler, mock(ContainerSignatureMatcher.class), mock(WebUIService.class), list, false, new HadoopShimsLoader(appContext.getAMConf()).getHadoopShim()) {

        @Override
        TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
            // Avoid wrapping in threads
            return rawContext;
        }
    };
    try {
        taskSchedulerManager.init(new TezConfiguration());
        taskSchedulerManager.start();
        taskSchedulerManager.getTotalResources(0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains(expIdentifier));
        reset(eventHandler);
        taskSchedulerManager.getAvailableResources(0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains(expIdentifier));
    } finally {
        taskSchedulerManager.stop();
    }
}

Example 45 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestDAGSchedulerNaturalOrderControlled method testSimpleFlow.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testSimpleFlow() {
    EventHandler eventHandler = mock(EventHandler.class);
    DAG dag = createMockDag();
    DAGSchedulerNaturalOrderControlled dagScheduler = new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
    int numVertices = 5;
    Vertex[] vertices = new Vertex[numVertices];
    for (int i = 0; i < numVertices; i++) {
        vertices[i] = dag.getVertex("vertex" + i);
    }
    // Schedule all tasks belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[0].getTotalTasks())).handle(any(Event.class));
    reset(eventHandler);
    // Schedule 3 tasks belonging to v2
    for (int i = 0; i < 3; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    verify(eventHandler, times(3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule 3 tasks belonging to v3
    for (int i = 0; i < 3; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[3].getVertexId(), i, 0));
    }
    verify(eventHandler, times(3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule remaining tasks belonging to v2
    for (int i = 3; i < vertices[2].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[2].getTotalTasks() - 3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule remaining tasks belonging to v3
    for (int i = 3; i < vertices[3].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[3].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[3].getTotalTasks() - 3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule all tasks belonging to v4
    for (int i = 0; i < vertices[4].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[4].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[4].getTotalTasks())).handle(any(Event.class));
    reset(eventHandler);
}