Examples with RMAppRunningOnNodeEvent org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent used on opensource projects

Example 1 with RMAppRunningOnNodeEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent in project hadoop by apache.

the class RMNodeImpl method handleRunningAppOnNode.

private static void handleRunningAppOnNode(RMNodeImpl rmNode, RMContext context, ApplicationId appId, NodeId nodeId) {
    RMApp app = context.getRMApps().get(appId);
    // cleaned up on the NM
    if (null == app) {
        LOG.warn("Cannot get RMApp by appId=" + appId + ", just added it to finishedApplications list for cleanup");
        rmNode.finishedApplications.add(appId);
        rmNode.runningApplications.remove(appId);
        return;
    }
    // Add running applications back due to Node add or Node reconnection.
    rmNode.runningApplications.add(appId);
    context.getDispatcher().getEventHandler().handle(new RMAppRunningOnNodeEvent(appId, nodeId));
}

Example 2 with RMAppRunningOnNodeEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testFinishedContainer.

@Test
public void testFinishedContainer() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    // Complete one container
    ContainerId containerId1 = BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 2);
    Container container1 = mock(Container.class);
    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
    when(container1.getId()).thenReturn(containerId1);
    when(containerStatus1.getContainerId()).thenReturn(containerId1);
    when(container1.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
    application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(), container1.getNodeId()));
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), containerStatus1, container1.getNodeId()));
    ArgumentCaptor<RMNodeFinishedContainersPulledByAMEvent> captor = ArgumentCaptor.forClass(RMNodeFinishedContainersPulledByAMEvent.class);
    // Verify justFinishedContainers
    Assert.assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
    Assert.assertEquals(container1.getId(), applicationAttempt.getJustFinishedContainers().get(0).getContainerId());
    Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt).size());
    // Verify finishedContainersSentToAM gets container after pull
    List<ContainerStatus> containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(1, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler, never()).handle(Mockito.any(RMNodeEvent.class));
    Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty());
    Assert.assertEquals(1, getFinishedContainersSentToAM(applicationAttempt).size());
    // Verify container is acked to NM via the RMNodeEvent after second pull
    containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(0, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler).handle(captor.capture());
    Assert.assertEquals(container1.getId(), captor.getValue().getContainers().get(0));
    Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty());
    Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt).size());
    // verify if no containers to acknowledge to NM then event should not be
    // triggered. Number of times event invoked is 1 i.e on second pull
    containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(0, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler, times(1)).handle(Mockito.any(RMNodeEvent.class));
}

Example 3 with RMAppRunningOnNodeEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testUnmanagedAMSuccess.

private void testUnmanagedAMSuccess(String url) {
    unmanagedAM = true;
    when(submissionContext.getUnmanagedAM()).thenReturn(true);
    // submit AM and check it goes to LAUNCHED state
    scheduleApplicationAttempt();
    testAppAttemptLaunchedState(null);
    verify(amLivelinessMonitor, times(1)).register(applicationAttempt.getAppAttemptId());
    // launch AM
    runApplicationAttempt(null, "host", 8042, url, true);
    // complete a container
    Container container = mock(Container.class);
    when(container.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
    application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(), container.getNodeId()));
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class), container.getNodeId()));
    // complete AM
    String diagnostics = "Successful";
    FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
    applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(applicationAttempt.getAppAttemptId(), url, finalStatus, diagnostics));
    sendAttemptUpdateSavedEvent(applicationAttempt);
    testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1, true);
    assertFalse(transferStateFromPreviousAttempt);
}

Example 4 with RMAppRunningOnNodeEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent in project hadoop by apache.

the class TestRMAppLogAggregationStatus method testLogAggregationStatus.

@Test
public void testLogAggregationStatus() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    conf.setLong(YarnConfiguration.LOG_AGGREGATION_STATUS_TIME_OUT_MS, 1500);
    RMApp rmApp = createRMApp(conf);
    this.rmContext.getRMApps().put(appId, rmApp);
    rmApp.handle(new RMAppEvent(this.appId, RMAppEventType.START));
    rmApp.handle(new RMAppEvent(this.appId, RMAppEventType.APP_NEW_SAVED));
    rmApp.handle(new RMAppEvent(this.appId, RMAppEventType.APP_ACCEPTED));
    // This application will be running on two nodes
    NodeId nodeId1 = NodeId.newInstance("localhost", 1234);
    Resource capability = Resource.newInstance(4096, 4);
    RMNodeImpl node1 = new RMNodeImpl(nodeId1, rmContext, null, 0, 0, null, capability, null);
    node1.handle(new RMNodeStartedEvent(nodeId1, null, null));
    rmApp.handle(new RMAppRunningOnNodeEvent(this.appId, nodeId1));
    NodeId nodeId2 = NodeId.newInstance("localhost", 2345);
    RMNodeImpl node2 = new RMNodeImpl(nodeId2, rmContext, null, 0, 0, null, capability, null);
    node2.handle(new RMNodeStartedEvent(node2.getNodeID(), null, null));
    rmApp.handle(new RMAppRunningOnNodeEvent(this.appId, nodeId2));
    // The initial log aggregation status for these two nodes
    // should be NOT_STARTED
    Map<NodeId, LogAggregationReport> logAggregationStatus = rmApp.getLogAggregationReportsForApp();
    Assert.assertEquals(2, logAggregationStatus.size());
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId1));
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId2));
    for (Entry<NodeId, LogAggregationReport> report : logAggregationStatus.entrySet()) {
        Assert.assertEquals(LogAggregationStatus.NOT_START, report.getValue().getLogAggregationStatus());
    }
    List<LogAggregationReport> node1ReportForApp = new ArrayList<LogAggregationReport>();
    String messageForNode1_1 = "node1 logAggregation status updated at " + System.currentTimeMillis();
    LogAggregationReport report1 = LogAggregationReport.newInstance(appId, LogAggregationStatus.RUNNING, messageForNode1_1);
    node1ReportForApp.add(report1);
    NodeStatus nodeStatus1 = NodeStatus.newInstance(node1.getNodeID(), 0, new ArrayList<ContainerStatus>(), null, NodeHealthStatus.newInstance(true, null, 0), null, null, null);
    node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null, node1ReportForApp));
    List<LogAggregationReport> node2ReportForApp = new ArrayList<LogAggregationReport>();
    String messageForNode2_1 = "node2 logAggregation status updated at " + System.currentTimeMillis();
    LogAggregationReport report2 = LogAggregationReport.newInstance(appId, LogAggregationStatus.RUNNING, messageForNode2_1);
    node2ReportForApp.add(report2);
    NodeStatus nodeStatus2 = NodeStatus.newInstance(node2.getNodeID(), 0, new ArrayList<ContainerStatus>(), null, NodeHealthStatus.newInstance(true, null, 0), null, null, null);
    node2.handle(new RMNodeStatusEvent(node2.getNodeID(), nodeStatus2, null, node2ReportForApp));
    // node1 and node2 has updated its log aggregation status
    // verify that the log aggregation status for node1, node2
    // has been changed
    logAggregationStatus = rmApp.getLogAggregationReportsForApp();
    Assert.assertEquals(2, logAggregationStatus.size());
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId1));
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId2));
    for (Entry<NodeId, LogAggregationReport> report : logAggregationStatus.entrySet()) {
        if (report.getKey().equals(node1.getNodeID())) {
            Assert.assertEquals(LogAggregationStatus.RUNNING, report.getValue().getLogAggregationStatus());
            Assert.assertEquals(messageForNode1_1, report.getValue().getDiagnosticMessage());
        } else if (report.getKey().equals(node2.getNodeID())) {
            Assert.assertEquals(LogAggregationStatus.RUNNING, report.getValue().getLogAggregationStatus());
            Assert.assertEquals(messageForNode2_1, report.getValue().getDiagnosticMessage());
        } else {
            // should not contain log aggregation report for other nodes
            Assert.fail("should not contain log aggregation report for other nodes");
        }
    }
    // node1 updates its log aggregation status again
    List<LogAggregationReport> node1ReportForApp2 = new ArrayList<LogAggregationReport>();
    String messageForNode1_2 = "node1 logAggregation status updated at " + System.currentTimeMillis();
    LogAggregationReport report1_2 = LogAggregationReport.newInstance(appId, LogAggregationStatus.RUNNING, messageForNode1_2);
    node1ReportForApp2.add(report1_2);
    node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null, node1ReportForApp2));
    // verify that the log aggregation status for node1
    // has been changed
    // verify that the log aggregation status for node2
    // does not change
    logAggregationStatus = rmApp.getLogAggregationReportsForApp();
    Assert.assertEquals(2, logAggregationStatus.size());
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId1));
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId2));
    for (Entry<NodeId, LogAggregationReport> report : logAggregationStatus.entrySet()) {
        if (report.getKey().equals(node1.getNodeID())) {
            Assert.assertEquals(LogAggregationStatus.RUNNING, report.getValue().getLogAggregationStatus());
            Assert.assertEquals(messageForNode1_1 + "\n" + messageForNode1_2, report.getValue().getDiagnosticMessage());
        } else if (report.getKey().equals(node2.getNodeID())) {
            Assert.assertEquals(LogAggregationStatus.RUNNING, report.getValue().getLogAggregationStatus());
            Assert.assertEquals(messageForNode2_1, report.getValue().getDiagnosticMessage());
        } else {
            // should not contain log aggregation report for other nodes
            Assert.fail("should not contain log aggregation report for other nodes");
        }
    }
    // kill the application
    rmApp.handle(new RMAppEvent(appId, RMAppEventType.KILL));
    rmApp.handle(new RMAppEvent(appId, RMAppEventType.ATTEMPT_KILLED));
    rmApp.handle(new RMAppEvent(appId, RMAppEventType.APP_UPDATE_SAVED));
    Assert.assertEquals(RMAppState.KILLED, rmApp.getState());
    // wait for 1500 ms
    Thread.sleep(1500);
    // the log aggregation status for both nodes should be changed
    // to TIME_OUT
    logAggregationStatus = rmApp.getLogAggregationReportsForApp();
    Assert.assertEquals(2, logAggregationStatus.size());
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId1));
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId2));
    for (Entry<NodeId, LogAggregationReport> report : logAggregationStatus.entrySet()) {
        Assert.assertEquals(LogAggregationStatus.TIME_OUT, report.getValue().getLogAggregationStatus());
    }
    // Finally, node1 finished its log aggregation and sent out its final
    // log aggregation status. The log aggregation status for node1 should
    // be changed from TIME_OUT to SUCCEEDED
    List<LogAggregationReport> node1ReportForApp3 = new ArrayList<LogAggregationReport>();
    LogAggregationReport report1_3;
    for (int i = 0; i < 10; i++) {
        report1_3 = LogAggregationReport.newInstance(appId, LogAggregationStatus.RUNNING, "test_message_" + i);
        node1ReportForApp3.add(report1_3);
    }
    node1ReportForApp3.add(LogAggregationReport.newInstance(appId, LogAggregationStatus.SUCCEEDED, ""));
    // For every logAggregationReport cached in memory, we can only save at most
    // 10 diagnostic messages/failure messages
    node1.handle(new RMNodeStatusEvent(node1.getNodeID(), nodeStatus1, null, node1ReportForApp3));
    logAggregationStatus = rmApp.getLogAggregationReportsForApp();
    Assert.assertEquals(2, logAggregationStatus.size());
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId1));
    Assert.assertTrue(logAggregationStatus.containsKey(nodeId2));
    for (Entry<NodeId, LogAggregationReport> report : logAggregationStatus.entrySet()) {
        if (report.getKey().equals(node1.getNodeID())) {
            Assert.assertEquals(LogAggregationStatus.SUCCEEDED, report.getValue().getLogAggregationStatus());
            StringBuilder builder = new StringBuilder();
            for (int i = 0; i < 9; i++) {
                builder.append("test_message_" + i);
                builder.append("\n");
            }
            builder.append("test_message_" + 9);
            Assert.assertEquals(builder.toString(), report.getValue().getDiagnosticMessage());
        } else if (report.getKey().equals(node2.getNodeID())) {
            Assert.assertEquals(LogAggregationStatus.TIME_OUT, report.getValue().getLogAggregationStatus());
        } else {
            // should not contain log aggregation report for other nodes
            Assert.fail("should not contain log aggregation report for other nodes");
        }
    }
    // update log aggregationStatus for node2 as FAILED,
    // so the log aggregation status for the App will become FAILED,
    // and we only keep the log aggregation reports whose status is FAILED,
    // so the log aggregation report for node1 will be removed.
    List<LogAggregationReport> node2ReportForApp2 = new ArrayList<LogAggregationReport>();
    LogAggregationReport report2_2 = LogAggregationReport.newInstance(appId, LogAggregationStatus.RUNNING_WITH_FAILURE, "Fail_Message");
    LogAggregationReport report2_3 = LogAggregationReport.newInstance(appId, LogAggregationStatus.FAILED, "");
    node2ReportForApp2.add(report2_2);
    node2ReportForApp2.add(report2_3);
    node2.handle(new RMNodeStatusEvent(node2.getNodeID(), nodeStatus2, null, node2ReportForApp2));
    Assert.assertEquals(LogAggregationStatus.FAILED, rmApp.getLogAggregationStatusForAppReport());
    logAggregationStatus = rmApp.getLogAggregationReportsForApp();
    Assert.assertTrue(logAggregationStatus.size() == 1);
    Assert.assertTrue(logAggregationStatus.containsKey(node2.getNodeID()));
    Assert.assertTrue(!logAggregationStatus.containsKey(node1.getNodeID()));
    Assert.assertEquals("Fail_Message", ((RMAppImpl) rmApp).getLogAggregationFailureMessagesForNM(nodeId2));
}