Examples with EventHandler org.apache.hadoop.yarn.event.EventHandler used on opensource projects

Example 6 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project hadoop by apache.

the class TestRM method testKillFinishingApp.

// Test Kill an app while the app is finishing in the meanwhile.
@Test(timeout = 30000)
public void testKillFinishingApp() throws Exception {
    // this dispatcher ignores RMAppAttemptEventType.KILL event
    final Dispatcher dispatcher = new DrainDispatcher() {

        @Override
        public EventHandler<Event> getEventHandler() {
            class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {

                @Override
                public boolean matches(Object argument) {
                    if (argument instanceof RMAppAttemptEvent) {
                        if (((RMAppAttemptEvent) argument).getType().equals(RMAppAttemptEventType.KILL)) {
                            return true;
                        }
                    }
                    return false;
                }
            }
            EventHandler handler = spy(super.getEventHandler());
            doNothing().when(handler).handle(argThat(new EventArgMatcher()));
            return handler;
        }
    };
    MockRM rm1 = new MockRM(conf) {

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    RMApp app1 = rm1.submitApp(200);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    rm1.killApp(app1.getApplicationId());
    FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "", "");
    am1.unregisterAppAttempt(req, true);
    rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHING);
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    rm1.waitForState(app1.getApplicationId(), RMAppState.FINISHED);
}

Example 7 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project hadoop by apache.

the class TestResourceTrackerService method testReconnectNode.

@Test
public void testReconnectNode() throws Exception {
    rm = new MockRM() {

        @Override
        protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
            return new EventDispatcher<SchedulerEvent>(this.scheduler, this.scheduler.getClass().getName()) {

                @Override
                public void handle(SchedulerEvent event) {
                    scheduler.handle(event);
                }
            };
        }
    };
    rm.start();
    MockNM nm1 = rm.registerNode("host1:1234", 5120);
    MockNM nm2 = rm.registerNode("host2:5678", 5120);
    nm1.nodeHeartbeat(true);
    nm2.nodeHeartbeat(false);
    rm.drainEvents();
    checkUnhealthyNMCount(rm, nm2, true, 1);
    final int expectedNMs = ClusterMetrics.getMetrics().getNumActiveNMs();
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    // TODO Metrics incorrect in case of the FifoScheduler
    Assert.assertEquals(5120, metrics.getAvailableMB());
    // reconnect of healthy node
    nm1 = rm.registerNode("host1:1234", 5120);
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    rm.drainEvents();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnhealthyNMCount(rm, nm2, true, 1);
    // reconnect of unhealthy node
    nm2 = rm.registerNode("host2:5678", 5120);
    response = nm2.nodeHeartbeat(false);
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    rm.drainEvents();
    Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs());
    checkUnhealthyNMCount(rm, nm2, true, 1);
    // unhealthy node changed back to healthy
    nm2 = rm.registerNode("host2:5678", 5120);
    response = nm2.nodeHeartbeat(true);
    response = nm2.nodeHeartbeat(true);
    rm.drainEvents();
    Assert.assertEquals(5120 + 5120, metrics.getAvailableMB());
    // reconnect of node with changed capability
    nm1 = rm.registerNode("host2:5678", 10240);
    response = nm1.nodeHeartbeat(true);
    rm.drainEvents();
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    Assert.assertEquals(5120 + 10240, metrics.getAvailableMB());
    // reconnect of node with changed capability and running applications
    List<ApplicationId> runningApps = new ArrayList<ApplicationId>();
    runningApps.add(ApplicationId.newInstance(1, 0));
    nm1 = rm.registerNode("host2:5678", 15360, 2, runningApps);
    response = nm1.nodeHeartbeat(true);
    rm.drainEvents();
    Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
    Assert.assertEquals(5120 + 15360, metrics.getAvailableMB());
    // reconnect healthy node changing http port
    nm1 = new MockNM("host1:1234", 5120, rm.getResourceTrackerService());
    nm1.setHttpPort(3);
    nm1.registerNode();
    response = nm1.nodeHeartbeat(true);
    response = nm1.nodeHeartbeat(true);
    rm.drainEvents();
    RMNode rmNode = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
    Assert.assertEquals(3, rmNode.getHttpPort());
    Assert.assertEquals(5120, rmNode.getTotalCapability().getMemorySize());
    Assert.assertEquals(5120 + 15360, metrics.getAvailableMB());
}

Example 8 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project hadoop by apache.

the class TestResourceTrackerService method testHandleContainerStatusInvalidCompletions.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testHandleContainerStatusInvalidCompletions() throws Exception {
    rm = new MockRM(new YarnConfiguration());
    rm.start();
    EventHandler handler = spy(rm.getRMContext().getDispatcher().getEventHandler());
    // Case 1: Unmanaged AM
    RMApp app = rm.submitApp(1024, true);
    // Case 1.1: AppAttemptId is null
    NMContainerStatus report = NMContainerStatus.newInstance(ContainerId.newContainerId(ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1), 0, ContainerState.COMPLETE, Resource.newInstance(1024, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234);
    rm.getResourceTrackerService().handleNMContainerStatus(report, null);
    verify(handler, never()).handle((Event) any());
    // Case 1.2: Master container is null
    RMAppAttemptImpl currentAttempt = (RMAppAttemptImpl) app.getCurrentAppAttempt();
    currentAttempt.setMasterContainer(null);
    report = NMContainerStatus.newInstance(ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0), 0, ContainerState.COMPLETE, Resource.newInstance(1024, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234);
    rm.getResourceTrackerService().handleNMContainerStatus(report, null);
    verify(handler, never()).handle((Event) any());
    // Case 2: Managed AM
    app = rm.submitApp(1024);
    // Case 2.1: AppAttemptId is null
    report = NMContainerStatus.newInstance(ContainerId.newContainerId(ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1), 0, ContainerState.COMPLETE, Resource.newInstance(1024, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234);
    try {
        rm.getResourceTrackerService().handleNMContainerStatus(report, null);
    } catch (Exception e) {
    // expected - ignore
    }
    verify(handler, never()).handle((Event) any());
    // Case 2.2: Master container is null
    currentAttempt = (RMAppAttemptImpl) app.getCurrentAppAttempt();
    currentAttempt.setMasterContainer(null);
    report = NMContainerStatus.newInstance(ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0), 0, ContainerState.COMPLETE, Resource.newInstance(1024, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234);
    try {
        rm.getResourceTrackerService().handleNMContainerStatus(report, null);
    } catch (Exception e) {
    // expected - ignore
    }
    verify(handler, never()).handle((Event) any());
}

Example 9 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project hadoop by apache.

the class TestUtils method getMockRMContext.

/**
   * Get a mock {@link RMContext} for use in test cases.
   * @return a mock {@link RMContext} for use in test cases
   */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static RMContext getMockRMContext() {
    // Null dispatcher
    Dispatcher nullDispatcher = new Dispatcher() {

        private final EventHandler handler = new EventHandler() {

            @Override
            public void handle(Event event) {
            }
        };

        @Override
        public void register(Class<? extends Enum> eventType, EventHandler handler) {
        }

        @Override
        public EventHandler<Event> getEventHandler() {
            return handler;
        }
    };
    // No op
    ContainerAllocationExpirer cae = new ContainerAllocationExpirer(nullDispatcher);
    Configuration conf = new Configuration();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    RMContextImpl rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null, new AMRMTokenSecretManager(conf, null), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM());
    RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
    when(nlm.getQueueResource(any(String.class), any(Set.class), any(Resource.class))).thenAnswer(new Answer<Resource>() {

        @Override
        public Resource answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            return (Resource) args[2];
        }
    });
    when(nlm.getResourceByLabel(any(String.class), any(Resource.class))).thenAnswer(new Answer<Resource>() {

        @Override
        public Resource answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            return (Resource) args[1];
        }
    });
    rmContext.setNodeLabelManager(nlm);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    ResourceScheduler mockScheduler = mock(ResourceScheduler.class);
    when(mockScheduler.getResourceCalculator()).thenReturn(new DefaultResourceCalculator());
    rmContext.setScheduler(mockScheduler);
    return rmContext;
}

Example 10 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project hadoop by apache.

the class TestRecovery method testRecoveryAllAttemptsKilled.

@Test
public void testRecoveryAllAttemptsKilled() {
    LOG.info("--- START:  testRecoveryAllAttemptsKilled ---");
    long clusterTimestamp = System.currentTimeMillis();
    EventHandler mockEventHandler = mock(EventHandler.class);
    MapTaskImpl recoverMapTask = getMockMapTask(clusterTimestamp, mockEventHandler);
    TaskId taskId = recoverMapTask.getID();
    JobID jobID = new JobID(Long.toString(clusterTimestamp), 1);
    TaskID taskID = new TaskID(jobID, org.apache.hadoop.mapreduce.TaskType.MAP, taskId.getId());
    //Mock up the TaskAttempts
    Map<TaskAttemptID, TaskAttemptInfo> mockTaskAttempts = new HashMap<TaskAttemptID, TaskAttemptInfo>();
    TaskAttemptID taId1 = new TaskAttemptID(taskID, 2);
    TaskAttemptInfo mockTAinfo1 = getMockTaskAttemptInfo(taId1, TaskAttemptState.KILLED);
    mockTaskAttempts.put(taId1, mockTAinfo1);
    TaskAttemptID taId2 = new TaskAttemptID(taskID, 1);
    TaskAttemptInfo mockTAinfo2 = getMockTaskAttemptInfo(taId2, TaskAttemptState.KILLED);
    mockTaskAttempts.put(taId2, mockTAinfo2);
    OutputCommitter mockCommitter = mock(OutputCommitter.class);
    TaskInfo mockTaskInfo = mock(TaskInfo.class);
    when(mockTaskInfo.getTaskStatus()).thenReturn("KILLED");
    when(mockTaskInfo.getTaskId()).thenReturn(taskID);
    when(mockTaskInfo.getAllTaskAttempts()).thenReturn(mockTaskAttempts);
    recoverMapTask.handle(new TaskRecoverEvent(taskId, mockTaskInfo, mockCommitter, true));
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    verify(mockEventHandler, atLeast(1)).handle((org.apache.hadoop.yarn.event.Event) arg.capture());
    Map<TaskAttemptID, TaskAttemptState> finalAttemptStates = new HashMap<TaskAttemptID, TaskAttemptState>();
    finalAttemptStates.put(taId1, TaskAttemptState.KILLED);
    finalAttemptStates.put(taId2, TaskAttemptState.KILLED);
    List<EventType> jobHistoryEvents = new ArrayList<EventType>();
    jobHistoryEvents.add(EventType.TASK_STARTED);
    jobHistoryEvents.add(EventType.MAP_ATTEMPT_STARTED);
    jobHistoryEvents.add(EventType.MAP_ATTEMPT_KILLED);
    jobHistoryEvents.add(EventType.MAP_ATTEMPT_STARTED);
    jobHistoryEvents.add(EventType.MAP_ATTEMPT_KILLED);
    jobHistoryEvents.add(EventType.TASK_FAILED);
    recoveryChecker(recoverMapTask, TaskState.KILLED, finalAttemptStates, arg, jobHistoryEvents, 2L, 0L);
}