Examples with RMNodeResourceUpdateEvent org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent used on opensource projects

Example 6 with RMNodeResourceUpdateEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent in project hadoop by apache.

the class TestCapacityScheduler method testResourceUpdateDecommissioningNode.

@Test
public void testResourceUpdateDecommissioningNode() throws Exception {
    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
    // to have 0 available resource
    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            if (event instanceof RMNodeResourceUpdateEvent) {
                RMNodeResourceUpdateEvent resourceEvent = (RMNodeResourceUpdateEvent) event;
                resourceManager.getResourceScheduler().getSchedulerNode(resourceEvent.getNodeId()).updateTotalResource(resourceEvent.getResourceOption().getResource());
            }
        }
    });
    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
    ((CapacityScheduler) resourceManager.getResourceScheduler()).setRMContext(spyContext);
    ((AsyncDispatcher) mockDispatcher).start();
    // Register node
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    // Submit an application
    Application application_0 = new Application("user_0", "a1", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_0);
    Task task_0_0 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    application_0.schedule();
    nodeUpdate(nm_0);
    // Kick off another heartbeat with the node state mocked to decommissioning
    // This should update the schedulernodes to have 0 available resource
    RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId()));
    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
    resourceManager.getResourceScheduler().handle(new NodeUpdateSchedulerEvent(spyNode));
    // Get allocations from the scheduler
    application_0.schedule();
    // Check the used resource is 1 GB 1 core
    Assert.assertEquals(1 * GB, nm_0.getUsed().getMemorySize());
    Resource usedResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
    Assert.assertEquals(usedResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(usedResource.getVirtualCores(), 1);
    // Check total resource of scheduler node is also changed to 1 GB 1 core
    Resource totalResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getTotalResource();
    Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(totalResource.getVirtualCores(), 1);
    // Check the available resource is 0/0
    Resource availableResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
    Assert.assertEquals(availableResource.getMemorySize(), 0);
    Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

Example 7 with RMNodeResourceUpdateEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent in project hadoop by apache.

the class AbstractYarnScheduler method nodeUpdate.

/**
   * Process a heartbeat update from a node.
   * @param nm The RMNode corresponding to the NodeManager
   */
protected void nodeUpdate(RMNode nm) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + getClusterResource());
    }
    // Process new container information
    List<ContainerStatus> completedContainers = updateNewContainerInfo(nm);
    // Process completed containers
    Resource releasedResources = Resource.newInstance(0, 0);
    int releasedContainers = updateCompletedContainers(completedContainers, releasedResources, nm.getNodeID());
    // update is propagated
    if (nm.getState() == NodeState.DECOMMISSIONING) {
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeResourceUpdateEvent(nm.getNodeID(), ResourceOption.newInstance(getSchedulerNode(nm.getNodeID()).getAllocatedResource(), 0)));
    }
    updateSchedulerHealthInformation(releasedResources, releasedContainers);
    updateNodeResourceUtilization(nm);
    // Now node data structures are up-to-date and ready for scheduling.
    if (LOG.isDebugEnabled()) {
        SchedulerNode node = getNode(nm.getNodeID());
        LOG.debug("Node being looked for scheduling " + nm + " availableResource: " + node.getUnallocatedResource());
    }
}

Example 8 with RMNodeResourceUpdateEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent in project hadoop by apache.

the class TestFairScheduler method testResourceUpdateDecommissioningNode.

@Test
public void testResourceUpdateDecommissioningNode() throws Exception {
    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
    // to have 0 available resource
    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler() {

        @Override
        public void handle(Event event) {
            if (event instanceof RMNodeResourceUpdateEvent) {
                RMNodeResourceUpdateEvent resourceEvent = (RMNodeResourceUpdateEvent) event;
                resourceManager.getResourceScheduler().getSchedulerNode(resourceEvent.getNodeId()).updateTotalResource(resourceEvent.getResourceOption().getResource());
            }
        }
    });
    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
    ((FairScheduler) resourceManager.getResourceScheduler()).setRMContext(spyContext);
    ((AsyncDispatcher) mockDispatcher).start();
    // Register node
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4));
    RMNode node = resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId());
    // Send a heartbeat to kick the tires on the Scheduler
    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
    resourceManager.getResourceScheduler().handle(nodeUpdate);
    // Kick off another heartbeat with the node state mocked to decommissioning
    // This should update the schedulernodes to have 0 available resource
    RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId()));
    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
    resourceManager.getResourceScheduler().handle(new NodeUpdateSchedulerEvent(spyNode));
    // Check the used resource is 0 GB 0 core
    // Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
    Resource usedResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
    Assert.assertEquals(usedResource.getMemorySize(), 0);
    Assert.assertEquals(usedResource.getVirtualCores(), 0);
    // Check total resource of scheduler node is also changed to 0 GB 0 core
    Resource totalResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getTotalResource();
    Assert.assertEquals(totalResource.getMemorySize(), 0 * GB);
    Assert.assertEquals(totalResource.getVirtualCores(), 0);
    // Check the available resource is 0/0
    Resource availableResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
    Assert.assertEquals(availableResource.getMemorySize(), 0);
    Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

Example 9 with RMNodeResourceUpdateEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent in project hadoop by apache.

the class AdminService method updateNodeResource.

@SuppressWarnings("unchecked")
@Override
public UpdateNodeResourceResponse updateNodeResource(UpdateNodeResourceRequest request) throws YarnException, IOException {
    final String operation = "updateNodeResource";
    UserGroupInformation user = checkAcls(operation);
    checkRMStatus(user.getShortUserName(), operation, "update node resource.");
    Map<NodeId, ResourceOption> nodeResourceMap = request.getNodeResourceMap();
    Set<NodeId> nodeIds = nodeResourceMap.keySet();
    // valid nodes.
    for (NodeId nodeId : nodeIds) {
        RMNode node = this.rmContext.getRMNodes().get(nodeId);
        if (node == null) {
            LOG.error("Resource update get failed on all nodes due to change " + "resource on an unrecognized node: " + nodeId);
            throw RPCUtil.getRemoteException("Resource update get failed on all nodes due to change resource " + "on an unrecognized node: " + nodeId);
        }
    }
    // do resource update on each node.
    // Notice: it is still possible to have invalid NodeIDs as nodes decommission
    // may happen just at the same time. This time, only log and skip absent
    // nodes without throwing any exceptions.
    boolean allSuccess = true;
    for (Map.Entry<NodeId, ResourceOption> entry : nodeResourceMap.entrySet()) {
        ResourceOption newResourceOption = entry.getValue();
        NodeId nodeId = entry.getKey();
        RMNode node = this.rmContext.getRMNodes().get(nodeId);
        if (node == null) {
            LOG.warn("Resource update get failed on an unrecognized node: " + nodeId);
            allSuccess = false;
        } else {
            // update resource to RMNode
            this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeResourceUpdateEvent(nodeId, newResourceOption));
            LOG.info("Update resource on node(" + node.getNodeID() + ") with resource(" + newResourceOption.toString() + ")");
        }
    }
    if (allSuccess) {
        RMAuditLogger.logSuccess(user.getShortUserName(), operation, "AdminService");
    }
    UpdateNodeResourceResponse response = UpdateNodeResourceResponse.newInstance();
    return response;
}