Examples with Allocation org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation used on opensource projects

Example 11 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class FifoScheduler method allocate.

@Override
public Allocation allocate(ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests) {
    FifoAppAttempt application = getApplicationAttempt(applicationAttemptId);
    if (application == null) {
        LOG.error("Calling allocate on removed " + "or non-existent application " + applicationAttemptId);
        return EMPTY_ALLOCATION;
    }
    // Sanity check
    normalizeRequests(ask);
    // Release containers
    releaseContainers(release, application);
    synchronized (application) {
        // when the allocate comes in
        if (application.isStopped()) {
            LOG.info("Calling allocate on a stopped " + "application " + applicationAttemptId);
            return EMPTY_ALLOCATION;
        }
        if (!ask.isEmpty()) {
            LOG.debug("allocate: pre-update" + " applicationId=" + applicationAttemptId + " application=" + application);
            application.showRequests();
            // Update application requests
            application.updateResourceRequests(ask);
            LOG.debug("allocate: post-update" + " applicationId=" + applicationAttemptId + " application=" + application);
            application.showRequests();
            LOG.debug("allocate:" + " applicationId=" + applicationAttemptId + " #ask=" + ask.size());
        }
        application.updateBlacklist(blacklistAdditions, blacklistRemovals);
        Resource headroom = application.getHeadroom();
        application.setApplicationHeadroomForMetrics(headroom);
        return new Allocation(application.pullNewlyAllocatedContainers(), headroom, null, null, null, application.pullUpdatedNMTokens());
    }
}

Example 12 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class FairScheduler method allocate.

@Override
public Allocation allocate(ApplicationAttemptId appAttemptId, List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests) {
    // Make sure this application exists
    FSAppAttempt application = getSchedulerApp(appAttemptId);
    if (application == null) {
        LOG.info("Calling allocate on removed " + "or non existent application " + appAttemptId);
        return EMPTY_ALLOCATION;
    }
    // Handle promotions and demotions
    handleContainerUpdates(application, updateRequests);
    // Sanity check
    normalizeRequests(ask);
    // Record container allocation start time
    application.recordContainerRequestTime(getClock().getTime());
    // Release containers
    releaseContainers(release, application);
    ReentrantReadWriteLock.WriteLock lock = application.getWriteLock();
    lock.lock();
    try {
        if (!ask.isEmpty()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("allocate: pre-update" + " applicationAttemptId=" + appAttemptId + " application=" + application.getApplicationId());
            }
            application.showRequests();
            // Update application requests
            application.updateResourceRequests(ask);
            application.showRequests();
        }
    } finally {
        lock.unlock();
    }
    Set<ContainerId> preemptionContainerIds = application.getPreemptionContainerIds();
    if (LOG.isDebugEnabled()) {
        LOG.debug("allocate: post-update" + " applicationAttemptId=" + appAttemptId + " #ask=" + ask.size() + " reservation= " + application.getCurrentReservation());
        LOG.debug("Preempting " + preemptionContainerIds.size() + " container(s)");
    }
    application.updateBlacklist(blacklistAdditions, blacklistRemovals);
    List<Container> newlyAllocatedContainers = application.pullNewlyAllocatedContainers();
    // Record container allocation time
    if (!(newlyAllocatedContainers.isEmpty())) {
        application.recordContainerAllocationTime(getClock().getTime());
    }
    Resource headroom = application.getHeadroom();
    application.setApplicationHeadroomForMetrics(headroom);
    return new Allocation(newlyAllocatedContainers, headroom, preemptionContainerIds, null, null, application.pullUpdatedNMTokens(), null, null, application.pullNewlyPromotedContainers(), application.pullNewlyDemotedContainers());
}

Example 13 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class TestFifoScheduler method testHeadroom.

@Test(timeout = 50000)
public void testHeadroom() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    // Add a node
    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
    fs.handle(new NodeAddedSchedulerEvent(n1));
    // Add two applications
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1);
    createMockRMApp(appAttemptId1, rm.getRMContext());
    SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
    fs.handle(appEvent);
    SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    fs.handle(attemptEvent);
    ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(appId2, 1);
    createMockRMApp(appAttemptId2, rm.getRMContext());
    SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user");
    fs.handle(appEvent2);
    SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
    fs.handle(attemptEvent2);
    List<ContainerId> emptyId = new ArrayList<ContainerId>();
    List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();
    // Set up resource requests
    // Ask for a 1 GB container for app 1
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
    fs.allocate(appAttemptId1, ask1, emptyId, null, null, NULL_UPDATE_REQUESTS);
    // Ask for a 2 GB container for app 2
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
    fs.allocate(appAttemptId2, ask2, emptyId, null, null, NULL_UPDATE_REQUESTS);
    // Trigger container assignment
    fs.handle(new NodeUpdateSchedulerEvent(n1));
    // Get the allocation for the applications and verify headroom
    Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("Allocation headroom", 1 * GB, allocation1.getResourceLimit().getMemorySize());
    Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("Allocation headroom", 1 * GB, allocation2.getResourceLimit().getMemorySize());
    rm.stop();
}

Example 14 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class ApplicationMasterService method allocateInternal.

protected void allocateInternal(ApplicationAttemptId appAttemptId, AllocateRequest request, AllocateResponse allocateResponse) throws YarnException {
    //filter illegal progress values
    float filteredProgress = request.getProgress();
    if (Float.isNaN(filteredProgress) || filteredProgress == Float.NEGATIVE_INFINITY || filteredProgress < 0) {
        request.setProgress(0);
    } else if (filteredProgress > 1 || filteredProgress == Float.POSITIVE_INFINITY) {
        request.setProgress(1);
    }
    // Send the status update to the appAttempt.
    this.rmContext.getDispatcher().getEventHandler().handle(new RMAppAttemptStatusupdateEvent(appAttemptId, request.getProgress()));
    List<ResourceRequest> ask = request.getAskList();
    List<ContainerId> release = request.getReleaseList();
    ResourceBlacklistRequest blacklistRequest = request.getResourceBlacklistRequest();
    List<String> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.getBlacklistAdditions() : Collections.EMPTY_LIST;
    List<String> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.getBlacklistRemovals() : Collections.EMPTY_LIST;
    RMApp app = this.rmContext.getRMApps().get(appAttemptId.getApplicationId());
    // set label expression for Resource Requests if resourceName=ANY
    ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
    for (ResourceRequest req : ask) {
        if (null == req.getNodeLabelExpression() && ResourceRequest.ANY.equals(req.getResourceName())) {
            req.setNodeLabelExpression(asc.getNodeLabelExpression());
        }
    }
    Resource maximumCapacity = rScheduler.getMaximumResourceCapability();
    // sanity check
    try {
        RMServerUtils.normalizeAndValidateRequests(ask, maximumCapacity, app.getQueue(), rScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        LOG.warn("Invalid resource ask by application " + appAttemptId, e);
        throw e;
    }
    try {
        RMServerUtils.validateBlacklistRequest(blacklistRequest);
    } catch (InvalidResourceBlacklistRequestException e) {
        LOG.warn("Invalid blacklist request by application " + appAttemptId, e);
        throw e;
    }
    // AM to release containers from the earlier attempt.
    if (!app.getApplicationSubmissionContext().getKeepContainersAcrossApplicationAttempts()) {
        try {
            RMServerUtils.validateContainerReleaseRequest(release, appAttemptId);
        } catch (InvalidContainerReleaseException e) {
            LOG.warn("Invalid container release by application " + appAttemptId, e);
            throw e;
        }
    }
    // Split Update Resource Requests into increase and decrease.
    // No Exceptions are thrown here. All update errors are aggregated
    // and returned to the AM.
    List<UpdateContainerError> updateErrors = new ArrayList<>();
    ContainerUpdates containerUpdateRequests = RMServerUtils.validateAndSplitUpdateResourceRequests(rmContext, request, maximumCapacity, updateErrors);
    // Send new requests to appAttempt.
    Allocation allocation;
    RMAppAttemptState state = app.getRMAppAttempt(appAttemptId).getAppAttemptState();
    if (state.equals(RMAppAttemptState.FINAL_SAVING) || state.equals(RMAppAttemptState.FINISHING) || app.isAppFinalStateStored()) {
        LOG.warn(appAttemptId + " is in " + state + " state, ignore container allocate request.");
        allocation = EMPTY_ALLOCATION;
    } else {
        allocation = this.rScheduler.allocate(appAttemptId, ask, release, blacklistAdditions, blacklistRemovals, containerUpdateRequests);
    }
    if (!blacklistAdditions.isEmpty() || !blacklistRemovals.isEmpty()) {
        LOG.info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals);
    }
    RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
    if (allocation.getNMTokens() != null && !allocation.getNMTokens().isEmpty()) {
        allocateResponse.setNMTokens(allocation.getNMTokens());
    }
    // Notify the AM of container update errors
    addToUpdateContainerErrors(allocateResponse, updateErrors);
    // update the response with the deltas of node status changes
    List<RMNode> updatedNodes = new ArrayList<RMNode>();
    if (app.pullRMNodeUpdates(updatedNodes) > 0) {
        List<NodeReport> updatedNodeReports = new ArrayList<NodeReport>();
        for (RMNode rmNode : updatedNodes) {
            SchedulerNodeReport schedulerNodeReport = rScheduler.getNodeReport(rmNode.getNodeID());
            Resource used = BuilderUtils.newResource(0, 0);
            int numContainers = 0;
            if (schedulerNodeReport != null) {
                used = schedulerNodeReport.getUsedResource();
                numContainers = schedulerNodeReport.getNumContainers();
            }
            NodeId nodeId = rmNode.getNodeID();
            NodeReport report = BuilderUtils.newNodeReport(nodeId, rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels());
            updatedNodeReports.add(report);
        }
        allocateResponse.setUpdatedNodes(updatedNodeReports);
    }
    addToAllocatedContainers(allocateResponse, allocation.getContainers());
    allocateResponse.setCompletedContainersStatuses(appAttempt.pullJustFinishedContainers());
    allocateResponse.setAvailableResources(allocation.getResourceLimit());
    addToContainerUpdates(appAttemptId, allocateResponse, allocation);
    allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
    // add collector address for this application
    if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
        allocateResponse.setCollectorAddr(this.rmContext.getRMApps().get(appAttemptId.getApplicationId()).getCollectorAddr());
    }
    // add preemption to the allocateResponse message (if any)
    allocateResponse.setPreemptionMessage(generatePreemptionMessage(allocation));
    // Set application priority
    allocateResponse.setApplicationPriority(app.getApplicationPriority());
}

Example 15 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class TestCheckpointPreemptionPolicy method generatePreemptionMessage.

private PreemptionMessage generatePreemptionMessage(Set<ContainerId> containerToPreempt, HashMap<ContainerId, Resource> resPerCont, Resource minimumAllocation, boolean strict) {
    Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(new HashSet<ContainerId>(containerToPreempt));
    containerToPreempt.clear();
    Resource tot = Resource.newInstance(0, 0);
    for (ContainerId c : currentContPreemption) {
        Resources.addTo(tot, resPerCont.get(c));
    }
    int numCont = (int) Math.ceil(tot.getMemorySize() / (double) minimumAllocation.getMemorySize());
    ResourceRequest rr = ResourceRequest.newInstance(Priority.newInstance(0), ResourceRequest.ANY, minimumAllocation, numCont);
    if (strict) {
        return generatePreemptionMessage(new Allocation(null, null, currentContPreemption, null, null));
    }
    return generatePreemptionMessage(new Allocation(null, null, null, currentContPreemption, Collections.singletonList(rr)));
}