Examples with RMNodeStatusEvent org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent used on opensource projects

Example 6 with RMNodeStatusEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent in project hadoop by apache.

the class TestRMWebServicesNodes method testNodesResourceUtilization.

@Test
public void testNodesResourceUtilization() throws JSONException, Exception {
    WebResource r = resource();
    RMNode rmnode1 = getRunningRMNode("h1", 1234, 5120);
    NodeId nodeId1 = rmnode1.getNodeID();
    RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nodeId1);
    NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, "test health report", System.currentTimeMillis());
    ResourceUtilization nodeResource = ResourceUtilization.newInstance(4096, 0, (float) 10.5);
    ResourceUtilization containerResource = ResourceUtilization.newInstance(2048, 0, (float) 5.05);
    NodeStatus nodeStatus = NodeStatus.newInstance(nodeId1, 0, new ArrayList<ContainerStatus>(), null, nodeHealth, containerResource, nodeResource, null);
    node.handle(new RMNodeStatusEvent(nodeId1, nodeStatus, null));
    rm.waitForState(nodeId1, NodeState.RUNNING);
    ClientResponse response = r.path("ws").path("v1").path("cluster").path("nodes").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject nodes = json.getJSONObject("nodes");
    assertEquals("incorrect number of elements", 1, nodes.length());
    JSONArray nodeArray = nodes.getJSONArray("node");
    assertEquals("incorrect number of elements", 1, nodeArray.length());
    JSONObject info = nodeArray.getJSONObject(0);
    // verify the resource utilization
    verifyNodeInfo(info, rmnode1);
}

Example 7 with RMNodeStatusEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent in project hadoop by apache.

the class TestRMWebServicesNodes method testNodesDefaultWithUnHealthyNode.

@Test
public void testNodesDefaultWithUnHealthyNode() throws JSONException, Exception {
    WebResource r = resource();
    getRunningRMNode("h1", 1234, 5120);
    // h2 will be in NEW state
    getNewRMNode("h2", 1235, 5121);
    RMNode node3 = getRunningRMNode("h3", 1236, 5122);
    NodeId nodeId3 = node3.getNodeID();
    RMNode node = rm.getRMContext().getRMNodes().get(nodeId3);
    NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false, "test health report", System.currentTimeMillis());
    NodeStatus nodeStatus = NodeStatus.newInstance(nodeId3, 1, new ArrayList<ContainerStatus>(), null, nodeHealth, null, null, null);
    ((RMNodeImpl) node).handle(new RMNodeStatusEvent(nodeId3, nodeStatus, null));
    rm.waitForState(nodeId3, NodeState.UNHEALTHY);
    ClientResponse response = r.path("ws").path("v1").path("cluster").path("nodes").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject nodes = json.getJSONObject("nodes");
    assertEquals("incorrect number of elements", 1, nodes.length());
    JSONArray nodeArray = nodes.getJSONArray("node");
    // 3 nodes, including the unhealthy node and the new node.
    assertEquals("incorrect number of elements", 3, nodeArray.length());
}

Example 8 with RMNodeStatusEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent in project hadoop by apache.

the class TestResourceTrackerService method testNodeHeartbeatForAppCollectorsMap.

@Test
public void testNodeHeartbeatForAppCollectorsMap() throws Exception {
    Configuration conf = new Configuration();
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    // set version to 2
    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
    // enable aux-service based timeline collectors
    conf.set(YarnConfiguration.NM_AUX_SERVICES, "timeline_collector");
    conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + "timeline_collector" + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
    conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class);
    rm = new MockRM(conf);
    rm.start();
    MockNM nm1 = rm.registerNode("host1:1234", 5120);
    MockNM nm2 = rm.registerNode("host2:1234", 2048);
    NodeHeartbeatResponse nodeHeartbeat1 = nm1.nodeHeartbeat(true);
    NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true);
    RMNodeImpl node1 = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm1.getNodeId());
    RMNodeImpl node2 = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm2.getNodeId());
    RMApp app1 = rm.submitApp(1024);
    String collectorAddr1 = "1.2.3.4:5";
    app1.setCollectorAddr(collectorAddr1);
    String collectorAddr2 = "5.4.3.2:1";
    RMApp app2 = rm.submitApp(1024);
    app2.setCollectorAddr(collectorAddr2);
    // Create a running container for app1 running on nm1
    ContainerId runningContainerId1 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(app1.getApplicationId(), 0), 0);
    ContainerStatus status1 = ContainerStatus.newInstance(runningContainerId1, ContainerState.RUNNING, "", 0);
    List<ContainerStatus> statusList = new ArrayList<ContainerStatus>();
    statusList.add(status1);
    NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, "", System.currentTimeMillis());
    NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0, statusList, null, nodeHealth, null, null, null);
    node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus, nodeHeartbeat1));
    Assert.assertEquals(1, node1.getRunningApps().size());
    Assert.assertEquals(app1.getApplicationId(), node1.getRunningApps().get(0));
    // Create a running container for app2 running on nm2
    ContainerId runningContainerId2 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(app2.getApplicationId(), 0), 0);
    ContainerStatus status2 = ContainerStatus.newInstance(runningContainerId2, ContainerState.RUNNING, "", 0);
    statusList = new ArrayList<ContainerStatus>();
    statusList.add(status2);
    nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0, statusList, null, nodeHealth, null, null, null);
    node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeStatus, nodeHeartbeat2));
    Assert.assertEquals(1, node2.getRunningApps().size());
    Assert.assertEquals(app2.getApplicationId(), node2.getRunningApps().get(0));
    nodeHeartbeat1 = nm1.nodeHeartbeat(true);
    Map<ApplicationId, String> map1 = nodeHeartbeat1.getAppCollectorsMap();
    Assert.assertEquals(1, map1.size());
    Assert.assertEquals(collectorAddr1, map1.get(app1.getApplicationId()));
    nodeHeartbeat2 = nm2.nodeHeartbeat(true);
    Map<ApplicationId, String> map2 = nodeHeartbeat2.getAppCollectorsMap();
    Assert.assertEquals(1, map2.size());
    Assert.assertEquals(collectorAddr2, map2.get(app2.getApplicationId()));
}

Example 9 with RMNodeStatusEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent in project hadoop by apache.

the class TestRMNodeTransitions method testContainerUpdate.

@Test(timeout = 5000)
public void testContainerUpdate() throws InterruptedException {
    //Start the node
    node.handle(new RMNodeStartedEvent(null, null, null));
    NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
    RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
    node2.handle(new RMNodeStartedEvent(null, null, null));
    ApplicationId app0 = BuilderUtils.newApplicationId(0, 0);
    ApplicationId app1 = BuilderUtils.newApplicationId(1, 1);
    ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(app0, 0), 0);
    ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(app1, 1), 1);
    ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(app1, 1), 2);
    rmContext.getRMApps().put(app0, Mockito.mock(RMApp.class));
    rmContext.getRMApps().put(app1, Mockito.mock(RMApp.class));
    RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent(null);
    RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent(null);
    RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent(null);
    ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
    ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
    ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);
    doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode1)).when(statusEventFromNode1).getContainers();
    node.handle(statusEventFromNode1);
    Assert.assertEquals(1, completedContainers.size());
    Assert.assertEquals(completedContainerIdFromNode1, completedContainers.get(0).getContainerId());
    completedContainers.clear();
    doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode2_1)).when(statusEventFromNode2_1).getContainers();
    doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2).getContainerId();
    doReturn(Collections.singletonList(containerStatusFromNode2_2)).when(statusEventFromNode2_2).getContainers();
    node2.setNextHeartBeat(false);
    node2.handle(statusEventFromNode2_1);
    node2.setNextHeartBeat(true);
    node2.handle(statusEventFromNode2_2);
    Assert.assertEquals(2, completedContainers.size());
    Assert.assertEquals(completedContainerIdFromNode2_1, completedContainers.get(0).getContainerId());
    Assert.assertEquals(completedContainerIdFromNode2_2, completedContainers.get(1).getContainerId());
}

Example 10 with RMNodeStatusEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent in project hadoop by apache.

the class TestRMNodeTransitions method testContainerExpire.

@Test
public void testContainerExpire() throws Exception {
    ContainerAllocationExpirer mockExpirer = mock(ContainerAllocationExpirer.class);
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    rmContext.getRMApps().put(appId, Mockito.mock(RMApp.class));
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1L);
    ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2L);
    AllocationExpirationInfo expirationInfo1 = new AllocationExpirationInfo(containerId1);
    AllocationExpirationInfo expirationInfo2 = new AllocationExpirationInfo(containerId2);
    mockExpirer.register(expirationInfo1);
    mockExpirer.register(expirationInfo2);
    verify(mockExpirer).register(expirationInfo1);
    verify(mockExpirer).register(expirationInfo2);
    ((RMContextImpl) rmContext).setContainerAllocationExpirer(mockExpirer);
    RMNodeImpl rmNode = getRunningNode();
    ContainerStatus status1 = ContainerStatus.newInstance(containerId1, ContainerState.RUNNING, "", 0);
    ContainerStatus status2 = ContainerStatus.newInstance(containerId2, ContainerState.COMPLETE, "", 0);
    List<ContainerStatus> statusList = new ArrayList<ContainerStatus>();
    statusList.add(status1);
    statusList.add(status2);
    RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent(statusList);
    rmNode.handle(statusEvent);
    verify(mockExpirer).unregister(expirationInfo1);
    verify(mockExpirer).unregister(expirationInfo2);
}