Examples with NodeStatus org.apache.hadoop.yarn.server.api.records.NodeStatus used on opensource projects

Example 11 with NodeStatus

use of org.apache.hadoop.yarn.server.api.records.NodeStatus in project hadoop by apache.

the class TestResourceTrackerService method testNodeHeartBeatWithLabels.

@Test
public void testNodeHeartBeatWithLabels() throws Exception {
    writeToHostsFile("host2");
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
    conf.set(YarnConfiguration.NODELABEL_CONFIGURATION_TYPE, YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);
    final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();
    rm = new MockRM(conf) {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            return nodeLabelsMgr;
        }
    };
    rm.start();
    // adding valid labels
    try {
        nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C"));
    } catch (IOException e) {
        Assert.fail("Caught Exception while intializing");
        e.printStackTrace();
    }
    // Registering of labels and other required info to RM
    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest registerReq = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = NodeId.newInstance("host2", 1234);
    Resource capability = BuilderUtils.newResource(1024, 1);
    registerReq.setResource(capability);
    registerReq.setNodeId(nodeId);
    registerReq.setHttpPort(1234);
    registerReq.setNMVersion(YarnVersionInfo.getVersion());
    // Node register label
    registerReq.setNodeLabels(toNodeLabelSet("A"));
    RegisterNodeManagerResponse registerResponse = resourceTrackerService.registerNodeManager(registerReq);
    // modification of labels during heartbeat
    NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
    // Node heartbeat label update
    heartbeatReq.setNodeLabels(toNodeLabelSet("B"));
    NodeStatus nodeStatusObject = getNodeStatusObject(nodeId);
    heartbeatReq.setNodeStatus(nodeStatusObject);
    heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
    heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
    NodeHeartbeatResponse nodeHeartbeatResponse = resourceTrackerService.nodeHeartbeat(heartbeatReq);
    Assert.assertEquals("InValid Node Labels were not accepted by RM", NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction());
    assertCollectionEquals(nodeLabelsMgr.getNodeLabels().get(nodeId), NodeLabelsUtils.convertToStringSet(heartbeatReq.getNodeLabels()));
    Assert.assertTrue("Valid Node Labels were not accepted by RM", nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
    // After modification of labels next heartbeat sends null informing no update
    Set<String> oldLabels = nodeLabelsMgr.getNodeLabels().get(nodeId);
    int responseId = nodeStatusObject.getResponseId();
    heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
    // Node heartbeat label update
    heartbeatReq.setNodeLabels(null);
    nodeStatusObject = getNodeStatusObject(nodeId);
    nodeStatusObject.setResponseId(responseId + 2);
    heartbeatReq.setNodeStatus(nodeStatusObject);
    heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
    heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
    nodeHeartbeatResponse = resourceTrackerService.nodeHeartbeat(heartbeatReq);
    Assert.assertEquals("InValid Node Labels were not accepted by RM", NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction());
    assertCollectionEquals(nodeLabelsMgr.getNodeLabels().get(nodeId), oldLabels);
    Assert.assertFalse("Node Labels should not accepted by RM", nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
    rm.stop();
}

Example 12 with NodeStatus

use of org.apache.hadoop.yarn.server.api.records.NodeStatus in project hadoop by apache.

the class TestResourceTrackerService method testNodeHeartbeatForAppCollectorsMap.

@Test
public void testNodeHeartbeatForAppCollectorsMap() throws Exception {
    Configuration conf = new Configuration();
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    // set version to 2
    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
    // enable aux-service based timeline collectors
    conf.set(YarnConfiguration.NM_AUX_SERVICES, "timeline_collector");
    conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + "timeline_collector" + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
    conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class);
    rm = new MockRM(conf);
    rm.start();
    MockNM nm1 = rm.registerNode("host1:1234", 5120);
    MockNM nm2 = rm.registerNode("host2:1234", 2048);
    NodeHeartbeatResponse nodeHeartbeat1 = nm1.nodeHeartbeat(true);
    NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true);
    RMNodeImpl node1 = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm1.getNodeId());
    RMNodeImpl node2 = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm2.getNodeId());
    RMApp app1 = rm.submitApp(1024);
    String collectorAddr1 = "1.2.3.4:5";
    app1.setCollectorAddr(collectorAddr1);
    String collectorAddr2 = "5.4.3.2:1";
    RMApp app2 = rm.submitApp(1024);
    app2.setCollectorAddr(collectorAddr2);
    // Create a running container for app1 running on nm1
    ContainerId runningContainerId1 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(app1.getApplicationId(), 0), 0);
    ContainerStatus status1 = ContainerStatus.newInstance(runningContainerId1, ContainerState.RUNNING, "", 0);
    List<ContainerStatus> statusList = new ArrayList<ContainerStatus>();
    statusList.add(status1);
    NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, "", System.currentTimeMillis());
    NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0, statusList, null, nodeHealth, null, null, null);
    node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus, nodeHeartbeat1));
    Assert.assertEquals(1, node1.getRunningApps().size());
    Assert.assertEquals(app1.getApplicationId(), node1.getRunningApps().get(0));
    // Create a running container for app2 running on nm2
    ContainerId runningContainerId2 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(app2.getApplicationId(), 0), 0);
    ContainerStatus status2 = ContainerStatus.newInstance(runningContainerId2, ContainerState.RUNNING, "", 0);
    statusList = new ArrayList<ContainerStatus>();
    statusList.add(status2);
    nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0, statusList, null, nodeHealth, null, null, null);
    node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeStatus, nodeHeartbeat2));
    Assert.assertEquals(1, node2.getRunningApps().size());
    Assert.assertEquals(app2.getApplicationId(), node2.getRunningApps().get(0));
    nodeHeartbeat1 = nm1.nodeHeartbeat(true);
    Map<ApplicationId, String> map1 = nodeHeartbeat1.getAppCollectorsMap();
    Assert.assertEquals(1, map1.size());
    Assert.assertEquals(collectorAddr1, map1.get(app1.getApplicationId()));
    nodeHeartbeat2 = nm2.nodeHeartbeat(true);
    Map<ApplicationId, String> map2 = nodeHeartbeat2.getAppCollectorsMap();
    Assert.assertEquals(1, map2.size());
    Assert.assertEquals(collectorAddr2, map2.get(app2.getApplicationId()));
}

Example 13 with NodeStatus

use of org.apache.hadoop.yarn.server.api.records.NodeStatus in project hadoop by apache.

the class TestRMNodeTransitions method testDecommissioningUnhealthy.

// Test unhealthy report on a decommissioning node will make it
// keep decommissioning as long as there's a running or keep alive app.
// Otherwise, it will go to decommissioned
@Test
public void testDecommissioningUnhealthy() {
    RMNodeImpl node = getDecommissioningNode();
    NodeHealthStatus status = NodeHealthStatus.newInstance(false, "sick", System.currentTimeMillis());
    List<ApplicationId> keepAliveApps = new ArrayList<>();
    keepAliveApps.add(BuilderUtils.newApplicationId(1, 1));
    NodeStatus nodeStatus = NodeStatus.newInstance(node.getNodeID(), 0, null, keepAliveApps, status, null, null, null);
    node.handle(new RMNodeStatusEvent(node.getNodeID(), nodeStatus, null));
    Assert.assertEquals(NodeState.DECOMMISSIONING, node.getState());
    nodeStatus.setKeepAliveApplications(null);
    node.handle(new RMNodeStatusEvent(node.getNodeID(), nodeStatus, null));
    Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
}

Example 14 with NodeStatus

use of org.apache.hadoop.yarn.server.api.records.NodeStatus in project hadoop by apache.

the class TestRMNodeTransitions method getUnhealthyNode.

private RMNodeImpl getUnhealthyNode() {
    RMNodeImpl node = getRunningNode();
    NodeHealthStatus status = NodeHealthStatus.newInstance(false, "sick", System.currentTimeMillis());
    NodeStatus nodeStatus = NodeStatus.newInstance(node.getNodeID(), 0, new ArrayList<ContainerStatus>(), null, status, null, null, null);
    node.handle(new RMNodeStatusEvent(node.getNodeID(), nodeStatus, null));
    Assert.assertEquals(NodeState.UNHEALTHY, node.getState());
    return node;
}

Example 15 with NodeStatus

use of org.apache.hadoop.yarn.server.api.records.NodeStatus in project hadoop by apache.

the class TestRMNodeTransitions method testUpdateHeartbeatResponseForAppLifeCycle.

@Test(timeout = 20000)
public void testUpdateHeartbeatResponseForAppLifeCycle() {
    RMNodeImpl node = getRunningNode();
    NodeId nodeId = node.getNodeID();
    ApplicationId runningAppId = BuilderUtils.newApplicationId(0, 1);
    rmContext.getRMApps().put(runningAppId, Mockito.mock(RMApp.class));
    // Create a running container
    ContainerId runningContainerId = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(runningAppId, 0), 0);
    ContainerStatus status = ContainerStatus.newInstance(runningContainerId, ContainerState.RUNNING, "", 0);
    List<ContainerStatus> statusList = new ArrayList<ContainerStatus>();
    statusList.add(status);
    NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, "", System.currentTimeMillis());
    NodeStatus nodeStatus = NodeStatus.newInstance(nodeId, 0, statusList, null, nodeHealth, null, null, null);
    node.handle(new RMNodeStatusEvent(nodeId, nodeStatus, null));
    Assert.assertEquals(1, node.getRunningApps().size());
    // Finish an application
    ApplicationId finishedAppId = runningAppId;
    node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId));
    Assert.assertEquals(1, node.getAppsToCleanup().size());
    Assert.assertEquals(0, node.getRunningApps().size());
}