Examples with ResourceUtilization org.apache.hadoop.yarn.api.records.ResourceUtilization used on opensource projects

Example 1 with ResourceUtilization

use of org.apache.hadoop.yarn.api.records.ResourceUtilization in project hadoop by apache.

the class TestYarnCLI method getNodeReports.

private List<NodeReport> getNodeReports(int noOfNodes, NodeState state, boolean emptyNodeLabel, boolean emptyResourceUtilization) {
    List<NodeReport> nodeReports = new ArrayList<NodeReport>();
    for (int i = 0; i < noOfNodes; i++) {
        Set<String> nodeLabels = null;
        if (!emptyNodeLabel) {
            // node labels is not ordered, but when we output it, it should be
            // ordered
            nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
        }
        NodeReport nodeReport = NodeReport.newInstance(NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888", "rack1", Records.newRecord(Resource.class), Records.newRecord(Resource.class), 0, "", 0, nodeLabels);
        if (!emptyResourceUtilization) {
            ResourceUtilization containersUtilization = ResourceUtilization.newInstance(1024, 2048, 4);
            ResourceUtilization nodeUtilization = ResourceUtilization.newInstance(2048, 4096, 8);
            nodeReport.setAggregatedContainersUtilization(containersUtilization);
            nodeReport.setNodeUtilization(nodeUtilization);
        }
        nodeReports.add(nodeReport);
    }
    return nodeReports;
}

Example 2 with ResourceUtilization

use of org.apache.hadoop.yarn.api.records.ResourceUtilization in project hadoop by apache.

the class TestMiniYarnClusterNodeUtilization method createNodeStatus.

/**
   * Create a NodeStatus test vector.
   * @param nodeId Node identifier.
   * @param responseId Response identifier.
   * @param containerPMem Virtual memory of the container.
   * @param containerVMem Physical memory of the container.
   * @param containerCPU CPU percentage of the container.
   * @param nodePMem Physical memory of the node.
   * @param nodeVMem Virtual memory of the node.
   * @param nodeCPU CPU percentage of the node.
   */
private NodeStatus createNodeStatus(NodeId nodeId, int responseId, int containerPMem, int containerVMem, float containerCPU, int nodePMem, int nodeVMem, float nodeCPU) {
    // Fake node status with fake utilization
    ResourceUtilization containersUtilization = ResourceUtilization.newInstance(containerPMem, containerVMem, containerCPU);
    ResourceUtilization nodeUtilization = ResourceUtilization.newInstance(nodePMem, nodeVMem, nodeCPU);
    NodeStatus status = NodeStatus.newInstance(nodeId, responseId, new ArrayList<ContainerStatus>(), null, NodeHealthStatus.newInstance(true, null, 0), containersUtilization, nodeUtilization, null);
    return status;
}

Example 3 with ResourceUtilization

use of org.apache.hadoop.yarn.api.records.ResourceUtilization in project hadoop by apache.

the class TestMiniYarnClusterNodeUtilization method verifySimulatedUtilization.

/**
   * Verify both the RMNode and SchedulerNode have been updated with the test
   * fixture utilization data.
   */
private void verifySimulatedUtilization() throws InterruptedException {
    ResourceManager rm = cluster.getResourceManager(0);
    RMContext rmContext = rm.getRMContext();
    ResourceUtilization containersUtilization = nodeStatus.getContainersUtilization();
    ResourceUtilization nodeUtilization = nodeStatus.getNodeUtilization();
    // We check if the nodeUtilization is up to date
    for (int i = 0; i < 100; i++) {
        for (RMNode ni : rmContext.getRMNodes().values()) {
            if (ni.getNodeUtilization() != null) {
                if (ni.getNodeUtilization().equals(nodeUtilization)) {
                    break;
                }
            }
        }
        Thread.sleep(100);
    }
    // Verify the data is readable from the RM and scheduler nodes
    for (RMNode ni : rmContext.getRMNodes().values()) {
        ResourceUtilization cu = ni.getAggregatedContainersUtilization();
        assertEquals("Containers Utillization not propagated to RMNode", containersUtilization, cu);
        ResourceUtilization nu = ni.getNodeUtilization();
        assertEquals("Node Utillization not propagated to RMNode", nodeUtilization, nu);
        SchedulerNode scheduler = rmContext.getScheduler().getSchedulerNode(ni.getNodeID());
        cu = scheduler.getAggregatedContainersUtilization();
        assertEquals("Containers Utillization not propagated to SchedulerNode", containersUtilization, cu);
        nu = scheduler.getNodeUtilization();
        assertEquals("Node Utillization not propagated to SchedulerNode", nodeUtilization, nu);
    }
}

Example 4 with ResourceUtilization

use of org.apache.hadoop.yarn.api.records.ResourceUtilization in project hadoop by apache.

the class TestRMWebServicesNodes method testNodesResourceUtilization.

@Test
public void testNodesResourceUtilization() throws JSONException, Exception {
    WebResource r = resource();
    RMNode rmnode1 = getRunningRMNode("h1", 1234, 5120);
    NodeId nodeId1 = rmnode1.getNodeID();
    RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes().get(nodeId1);
    NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, "test health report", System.currentTimeMillis());
    ResourceUtilization nodeResource = ResourceUtilization.newInstance(4096, 0, (float) 10.5);
    ResourceUtilization containerResource = ResourceUtilization.newInstance(2048, 0, (float) 5.05);
    NodeStatus nodeStatus = NodeStatus.newInstance(nodeId1, 0, new ArrayList<ContainerStatus>(), null, nodeHealth, containerResource, nodeResource, null);
    node.handle(new RMNodeStatusEvent(nodeId1, nodeStatus, null));
    rm.waitForState(nodeId1, NodeState.RUNNING);
    ClientResponse response = r.path("ws").path("v1").path("cluster").path("nodes").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject nodes = json.getJSONObject("nodes");
    assertEquals("incorrect number of elements", 1, nodes.length());
    JSONArray nodeArray = nodes.getJSONArray("node");
    assertEquals("incorrect number of elements", 1, nodeArray.length());
    JSONObject info = nodeArray.getJSONObject(0);
    // verify the resource utilization
    verifyNodeInfo(info, rmnode1);
}

Example 5 with ResourceUtilization

use of org.apache.hadoop.yarn.api.records.ResourceUtilization in project hadoop by apache.

the class TestRMWebServicesNodes method verifyNodeInfoGeneric.

public void verifyNodeInfoGeneric(RMNode node, String state, String rack, String id, String nodeHostName, String nodeHTTPAddress, long lastHealthUpdate, String healthReport, int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores, long availVirtualCores, String version, int nodePhysicalMemoryMB, int nodeVirtualMemoryMB, double nodeCPUUsage, int containersPhysicalMemoryMB, int containersVirtualMemoryMB, double containersCPUUsage, int numRunningOpportContainers, long usedMemoryOpportGB, int usedVirtualCoresOpport, int numQueuedContainers) throws JSONException, Exception {
    ResourceScheduler sched = rm.getResourceScheduler();
    SchedulerNodeReport report = sched.getNodeReport(node.getNodeID());
    OpportunisticContainersStatus opportunisticStatus = node.getOpportunisticContainersStatus();
    WebServicesTestUtils.checkStringMatch("state", node.getState().toString(), state);
    WebServicesTestUtils.checkStringMatch("rack", node.getRackName(), rack);
    WebServicesTestUtils.checkStringMatch("id", node.getNodeID().toString(), id);
    WebServicesTestUtils.checkStringMatch("nodeHostName", node.getNodeID().getHost(), nodeHostName);
    WebServicesTestUtils.checkStringMatch("healthReport", String.valueOf(node.getHealthReport()), healthReport);
    String expectedHttpAddress = node.getNodeID().getHost() + ":" + node.getHttpPort();
    WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", expectedHttpAddress, nodeHTTPAddress);
    WebServicesTestUtils.checkStringMatch("version", node.getNodeManagerVersion(), version);
    if (node.getNodeUtilization() != null) {
        ResourceUtilization nodeResource = ResourceUtilization.newInstance(nodePhysicalMemoryMB, nodeVirtualMemoryMB, (float) nodeCPUUsage);
        assertEquals("nodeResourceUtilization doesn't match", node.getNodeUtilization(), nodeResource);
    }
    if (node.getAggregatedContainersUtilization() != null) {
        ResourceUtilization containerResource = ResourceUtilization.newInstance(containersPhysicalMemoryMB, containersVirtualMemoryMB, (float) containersCPUUsage);
        assertEquals("containerResourceUtilization doesn't match", node.getAggregatedContainersUtilization(), containerResource);
    }
    long expectedHealthUpdate = node.getLastHealthReportTime();
    assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate + " expected: " + expectedHealthUpdate, expectedHealthUpdate, lastHealthUpdate);
    if (report != null) {
        assertEquals("numContainers doesn't match: " + numContainers, report.getNumContainers(), numContainers);
        assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report.getUsedResource().getMemorySize(), usedMemoryMB);
        assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report.getAvailableResource().getMemorySize(), availMemoryMB);
        assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report.getUsedResource().getVirtualCores(), usedVirtualCores);
        assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report.getAvailableResource().getVirtualCores(), availVirtualCores);
    }
    if (opportunisticStatus != null) {
        assertEquals("numRunningOpportContainers doesn't match: " + numRunningOpportContainers, opportunisticStatus.getRunningOpportContainers(), numRunningOpportContainers);
        assertEquals("usedMemoryOpportGB doesn't match: " + usedMemoryOpportGB, opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpportGB);
        assertEquals("usedVirtualCoresOpport doesn't match: " + usedVirtualCoresOpport, opportunisticStatus.getOpportCoresUsed(), usedVirtualCoresOpport);
        assertEquals("numQueuedContainers doesn't match: " + numQueuedContainers, opportunisticStatus.getQueuedOpportContainers(), numQueuedContainers);
    }
}