Examples with SchedulerNodeReport org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport used on opensource projects

Example 11 with SchedulerNodeReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport in project hadoop by apache.

the class TestFifoScheduler method testMinimumAllocation.

private void testMinimumAllocation(YarnConfiguration conf, int testAlloc) throws Exception {
    MockRM rm = new MockRM(conf);
    rm.start();
    // Register node1
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    // Submit an application
    RMApp app1 = rm.submitApp(testAlloc);
    // kick the scheduling
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    am1.registerAppAttempt();
    SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
    int checkAlloc = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
    Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize());
    rm.stop();
}

Example 12 with SchedulerNodeReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport in project hadoop by apache.

the class TestNodeLabelContainerAllocation method testQueueMetricsWithLabels.

@Test
public void testQueueMetricsWithLabels() throws Exception {
    /**
     * Test case: have a following queue structure:
     *
     * <pre>
     *            root
     *         /      \
     *        a        b
     *        (x)     (x)
     * </pre>
     *
     * a/b can access x, both of them has max-capacity-on-x = 50
     *
     * When doing non-exclusive allocation, app in a (or b) can use 100% of x
     * resource.
     */
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(this.conf);
    // Define top-level queues
    csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" });
    csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
    final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
    csConf.setCapacity(queueA, 25);
    csConf.setAccessibleNodeLabels(queueA, toSet("x"));
    csConf.setCapacityByLabel(queueA, "x", 50);
    csConf.setMaximumCapacityByLabel(queueA, "x", 50);
    final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
    csConf.setCapacity(queueB, 75);
    csConf.setAccessibleNodeLabels(queueB, toSet("x"));
    csConf.setCapacityByLabel(queueB, "x", 50);
    csConf.setMaximumCapacityByLabel(queueB, "x", 50);
    // set node -> label
    mgr.addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x", false)));
    mgr.addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("y", false)));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
    // inject node label manager
    MockRM rm1 = new MockRM(csConf) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm1.getRMContext().setNodeLabelManager(mgr);
    rm1.start();
    // label = x
    MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB);
    // label = y
    MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB);
    // app1 -> a
    RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a", "x");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    // app1 asks for 5 partition=x containers
    am1.allocate("*", 1 * GB, 5, new ArrayList<ContainerId>(), "x");
    // NM1 do 50 heartbeats
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
    SchedulerNode schedulerNode1 = cs.getSchedulerNode(nm1.getNodeId());
    for (int i = 0; i < 50; i++) {
        cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
    }
    // app1 gets all resource in partition=x
    Assert.assertEquals(5, schedulerNode1.getNumContainers());
    SchedulerNodeReport reportNm1 = rm1.getResourceScheduler().getNodeReport(nm1.getNodeId());
    Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
    Assert.assertEquals(5 * GB, reportNm1.getAvailableResource().getMemorySize());
    SchedulerNodeReport reportNm2 = rm1.getResourceScheduler().getNodeReport(nm2.getNodeId());
    Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
    Assert.assertEquals(10 * GB, reportNm2.getAvailableResource().getMemorySize());
    LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
    assertEquals(0 * GB, leafQueue.getMetrics().getAvailableMB());
    assertEquals(5 * GB, leafQueue.getMetrics().getAllocatedMB());
    // Kill all apps in queue a
    cs.killAllAppsInQueue("a");
    rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
    rm1.waitForAppRemovedFromScheduler(app1.getApplicationId());
    assertEquals(0 * GB, leafQueue.getMetrics().getUsedAMResourceMB());
    assertEquals(0, leafQueue.getMetrics().getUsedAMResourceVCores());
    rm1.close();
}

Example 13 with SchedulerNodeReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport in project hadoop by apache.

the class ApplicationMasterService method allocateInternal.

protected void allocateInternal(ApplicationAttemptId appAttemptId, AllocateRequest request, AllocateResponse allocateResponse) throws YarnException {
    //filter illegal progress values
    float filteredProgress = request.getProgress();
    if (Float.isNaN(filteredProgress) || filteredProgress == Float.NEGATIVE_INFINITY || filteredProgress < 0) {
        request.setProgress(0);
    } else if (filteredProgress > 1 || filteredProgress == Float.POSITIVE_INFINITY) {
        request.setProgress(1);
    }
    // Send the status update to the appAttempt.
    this.rmContext.getDispatcher().getEventHandler().handle(new RMAppAttemptStatusupdateEvent(appAttemptId, request.getProgress()));
    List<ResourceRequest> ask = request.getAskList();
    List<ContainerId> release = request.getReleaseList();
    ResourceBlacklistRequest blacklistRequest = request.getResourceBlacklistRequest();
    List<String> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.getBlacklistAdditions() : Collections.EMPTY_LIST;
    List<String> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.getBlacklistRemovals() : Collections.EMPTY_LIST;
    RMApp app = this.rmContext.getRMApps().get(appAttemptId.getApplicationId());
    // set label expression for Resource Requests if resourceName=ANY
    ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
    for (ResourceRequest req : ask) {
        if (null == req.getNodeLabelExpression() && ResourceRequest.ANY.equals(req.getResourceName())) {
            req.setNodeLabelExpression(asc.getNodeLabelExpression());
        }
    }
    Resource maximumCapacity = rScheduler.getMaximumResourceCapability();
    // sanity check
    try {
        RMServerUtils.normalizeAndValidateRequests(ask, maximumCapacity, app.getQueue(), rScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        LOG.warn("Invalid resource ask by application " + appAttemptId, e);
        throw e;
    }
    try {
        RMServerUtils.validateBlacklistRequest(blacklistRequest);
    } catch (InvalidResourceBlacklistRequestException e) {
        LOG.warn("Invalid blacklist request by application " + appAttemptId, e);
        throw e;
    }
    // AM to release containers from the earlier attempt.
    if (!app.getApplicationSubmissionContext().getKeepContainersAcrossApplicationAttempts()) {
        try {
            RMServerUtils.validateContainerReleaseRequest(release, appAttemptId);
        } catch (InvalidContainerReleaseException e) {
            LOG.warn("Invalid container release by application " + appAttemptId, e);
            throw e;
        }
    }
    // Split Update Resource Requests into increase and decrease.
    // No Exceptions are thrown here. All update errors are aggregated
    // and returned to the AM.
    List<UpdateContainerError> updateErrors = new ArrayList<>();
    ContainerUpdates containerUpdateRequests = RMServerUtils.validateAndSplitUpdateResourceRequests(rmContext, request, maximumCapacity, updateErrors);
    // Send new requests to appAttempt.
    Allocation allocation;
    RMAppAttemptState state = app.getRMAppAttempt(appAttemptId).getAppAttemptState();
    if (state.equals(RMAppAttemptState.FINAL_SAVING) || state.equals(RMAppAttemptState.FINISHING) || app.isAppFinalStateStored()) {
        LOG.warn(appAttemptId + " is in " + state + " state, ignore container allocate request.");
        allocation = EMPTY_ALLOCATION;
    } else {
        allocation = this.rScheduler.allocate(appAttemptId, ask, release, blacklistAdditions, blacklistRemovals, containerUpdateRequests);
    }
    if (!blacklistAdditions.isEmpty() || !blacklistRemovals.isEmpty()) {
        LOG.info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals);
    }
    RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
    if (allocation.getNMTokens() != null && !allocation.getNMTokens().isEmpty()) {
        allocateResponse.setNMTokens(allocation.getNMTokens());
    }
    // Notify the AM of container update errors
    addToUpdateContainerErrors(allocateResponse, updateErrors);
    // update the response with the deltas of node status changes
    List<RMNode> updatedNodes = new ArrayList<RMNode>();
    if (app.pullRMNodeUpdates(updatedNodes) > 0) {
        List<NodeReport> updatedNodeReports = new ArrayList<NodeReport>();
        for (RMNode rmNode : updatedNodes) {
            SchedulerNodeReport schedulerNodeReport = rScheduler.getNodeReport(rmNode.getNodeID());
            Resource used = BuilderUtils.newResource(0, 0);
            int numContainers = 0;
            if (schedulerNodeReport != null) {
                used = schedulerNodeReport.getUsedResource();
                numContainers = schedulerNodeReport.getNumContainers();
            }
            NodeId nodeId = rmNode.getNodeID();
            NodeReport report = BuilderUtils.newNodeReport(nodeId, rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels());
            updatedNodeReports.add(report);
        }
        allocateResponse.setUpdatedNodes(updatedNodeReports);
    }
    addToAllocatedContainers(allocateResponse, allocation.getContainers());
    allocateResponse.setCompletedContainersStatuses(appAttempt.pullJustFinishedContainers());
    allocateResponse.setAvailableResources(allocation.getResourceLimit());
    addToContainerUpdates(appAttemptId, allocateResponse, allocation);
    allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
    // add collector address for this application
    if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
        allocateResponse.setCollectorAddr(this.rmContext.getRMApps().get(appAttemptId.getApplicationId()).getCollectorAddr());
    }
    // add preemption to the allocateResponse message (if any)
    allocateResponse.setPreemptionMessage(generatePreemptionMessage(allocation));
    // Set application priority
    allocateResponse.setApplicationPriority(app.getApplicationPriority());
}

Example 14 with SchedulerNodeReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport in project hadoop by apache.

the class ClientRMService method createNodeReports.

private NodeReport createNodeReports(RMNode rmNode) {
    SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID());
    Resource used = BuilderUtils.newResource(0, 0);
    int numContainers = 0;
    if (schedulerNodeReport != null) {
        used = schedulerNodeReport.getUsedResource();
        numContainers = schedulerNodeReport.getNumContainers();
    }
    NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(), rmNode.getNodeUtilization());
    return report;
}