Examples with Resource org.apache.hadoop.yarn.api.records.Resource used on opensource projects

Example 96 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class FSAppAttempt method fairShareStarvation.

/**
   * Helper method that computes the extent of fairshare starvation.
   * @return freshly computed fairshare starvation
   */
Resource fairShareStarvation() {
    long now = scheduler.getClock().getTime();
    Resource threshold = Resources.multiply(getFairShare(), fsQueue.getFairSharePreemptionThreshold());
    Resource fairDemand = Resources.componentwiseMin(threshold, demand);
    // Check if the queue is starved for fairshare
    boolean starved = isUsageBelowShare(getResourceUsage(), fairDemand);
    if (!starved) {
        lastTimeAtFairShare = now;
    }
    if (!starved || now - lastTimeAtFairShare < fsQueue.getFairSharePreemptionTimeout()) {
        fairshareStarvation = Resources.none();
    } else {
        // The app has been starved for longer than preemption-timeout.
        fairshareStarvation = Resources.subtractFromNonNegative(fairDemand, getResourceUsage());
    }
    return fairshareStarvation;
}

Example 97 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class FifoScheduler method allocate.

@Override
public Allocation allocate(ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests) {
    FifoAppAttempt application = getApplicationAttempt(applicationAttemptId);
    if (application == null) {
        LOG.error("Calling allocate on removed " + "or non-existent application " + applicationAttemptId);
        return EMPTY_ALLOCATION;
    }
    // Sanity check
    normalizeRequests(ask);
    // Release containers
    releaseContainers(release, application);
    synchronized (application) {
        // when the allocate comes in
        if (application.isStopped()) {
            LOG.info("Calling allocate on a stopped " + "application " + applicationAttemptId);
            return EMPTY_ALLOCATION;
        }
        if (!ask.isEmpty()) {
            LOG.debug("allocate: pre-update" + " applicationId=" + applicationAttemptId + " application=" + application);
            application.showRequests();
            // Update application requests
            application.updateResourceRequests(ask);
            LOG.debug("allocate: post-update" + " applicationId=" + applicationAttemptId + " application=" + application);
            application.showRequests();
            LOG.debug("allocate:" + " applicationId=" + applicationAttemptId + " #ask=" + ask.size());
        }
        application.updateBlacklist(blacklistAdditions, blacklistRemovals);
        Resource headroom = application.getHeadroom();
        application.setApplicationHeadroomForMetrics(headroom);
        return new Allocation(application.pullNewlyAllocatedContainers(), headroom, null, null, null, application.pullUpdatedNMTokens());
    }
}

Example 98 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class DominantResourceFairnessPolicy method getHeadroom.

@Override
public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, Resource maxAvailable) {
    long queueAvailableMemory = Math.max(queueFairShare.getMemorySize() - queueUsage.getMemorySize(), 0);
    int queueAvailableCPU = Math.max(queueFairShare.getVirtualCores() - queueUsage.getVirtualCores(), 0);
    Resource headroom = Resources.createResource(Math.min(maxAvailable.getMemorySize(), queueAvailableMemory), Math.min(maxAvailable.getVirtualCores(), queueAvailableCPU));
    return headroom;
}

Example 99 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class FairScheduler method applyChildDefaults.

/**
   * After reloading the allocation config, the max resource settings for any
   * ad hoc queues will be missing. This method goes through the queue manager's
   * queue list and adds back the max resources settings for any ad hoc queues.
   * Note that the new max resource settings will be based on the new config.
   * The old settings are lost.
   */
private void applyChildDefaults() {
    Collection<FSQueue> queues = queueMgr.getQueues();
    Set<String> configuredLeafQueues = allocConf.getConfiguredQueues().get(FSQueueType.LEAF);
    Set<String> configuredParentQueues = allocConf.getConfiguredQueues().get(FSQueueType.PARENT);
    for (FSQueue queue : queues) {
        // If the queue is ad hoc and not root, apply the child defaults
        if ((queue.getParent() != null) && !configuredLeafQueues.contains(queue.getName()) && !configuredParentQueues.contains(queue.getName())) {
            Resource max = queue.getParent().getMaxChildQueueResource();
            if (max != null) {
                queue.setMaxShare(max);
            }
        }
    }
}

Example 100 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class FairScheduler method allocate.

@Override
public Allocation allocate(ApplicationAttemptId appAttemptId, List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests) {
    // Make sure this application exists
    FSAppAttempt application = getSchedulerApp(appAttemptId);
    if (application == null) {
        LOG.info("Calling allocate on removed " + "or non existent application " + appAttemptId);
        return EMPTY_ALLOCATION;
    }
    // Handle promotions and demotions
    handleContainerUpdates(application, updateRequests);
    // Sanity check
    normalizeRequests(ask);
    // Record container allocation start time
    application.recordContainerRequestTime(getClock().getTime());
    // Release containers
    releaseContainers(release, application);
    ReentrantReadWriteLock.WriteLock lock = application.getWriteLock();
    lock.lock();
    try {
        if (!ask.isEmpty()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("allocate: pre-update" + " applicationAttemptId=" + appAttemptId + " application=" + application.getApplicationId());
            }
            application.showRequests();
            // Update application requests
            application.updateResourceRequests(ask);
            application.showRequests();
        }
    } finally {
        lock.unlock();
    }
    Set<ContainerId> preemptionContainerIds = application.getPreemptionContainerIds();
    if (LOG.isDebugEnabled()) {
        LOG.debug("allocate: post-update" + " applicationAttemptId=" + appAttemptId + " #ask=" + ask.size() + " reservation= " + application.getCurrentReservation());
        LOG.debug("Preempting " + preemptionContainerIds.size() + " container(s)");
    }
    application.updateBlacklist(blacklistAdditions, blacklistRemovals);
    List<Container> newlyAllocatedContainers = application.pullNewlyAllocatedContainers();
    // Record container allocation time
    if (!(newlyAllocatedContainers.isEmpty())) {
        application.recordContainerAllocationTime(getClock().getTime());
    }
    Resource headroom = application.getHeadroom();
    application.setApplicationHeadroomForMetrics(headroom);
    return new Allocation(newlyAllocatedContainers, headroom, preemptionContainerIds, null, null, application.pullUpdatedNMTokens(), null, null, application.pullNewlyPromotedContainers(), application.pullNewlyDemotedContainers());
}