Examples with Resource org.apache.hadoop.yarn.api.records.Resource used on opensource projects

Example 51 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class QueuePriorityContainerCandidateSelector method incToPreempt.

private void incToPreempt(String queue, String partition, Resource allocated) {
    Resource total = toPreemptedFromOtherQueues.get(queue, partition);
    if (null == total) {
        total = Resources.createResource(0);
        toPreemptedFromOtherQueues.put(queue, partition, total);
    }
    Resources.addTo(total, allocated);
}

Example 52 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class QueuePriorityContainerCandidateSelector method isQueueSatisfied.

/**
   * Do we allow the demanding queue preempt resource from other queues?
   * A satisfied queue is not allowed to preempt resource from other queues.
   * @param demandingQueue
   * @return allowed/not
   */
private boolean isQueueSatisfied(String demandingQueue, String partition) {
    TempQueuePerPartition tq = preemptionContext.getQueueByPartition(demandingQueue, partition);
    if (null == tq) {
        return false;
    }
    Resource guaranteed = tq.getGuaranteed();
    Resource usedDeductReservd = Resources.subtract(tq.getUsed(), tq.getReserved());
    Resource markedToPreemptFromOtherQueue = toPreemptedFromOtherQueues.get(demandingQueue, partition);
    if (null == markedToPreemptFromOtherQueue) {
        markedToPreemptFromOtherQueue = Resources.none();
    }
    // return Used - reserved + to-preempt-from-other-queue >= guaranteed
    boolean flag = Resources.greaterThanOrEqual(rc, clusterResource, Resources.add(usedDeductReservd, markedToPreemptFromOtherQueue), guaranteed);
    return flag;
}

Example 53 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ReservedContainerCandidatesSelector method getPreemptionCandidatesOnNode.

/**
   * Try to check if we can preempt resources for reserved container in given node
   * @param node
   * @param cluster
   * @param queueToPreemptableResourceByPartition it's a map of
   *                 <queueName, <partition, preemptable-resource>>
   * @param readOnly do we want to modify preemptable resource after we selected
   *                 candidates
   * @return NodeForPreemption if it's possible to preempt containers on the node
   * to satisfy reserved resource
   */
private NodeForPreemption getPreemptionCandidatesOnNode(FiCaSchedulerNode node, Resource cluster, Map<String, Map<String, Resource>> queueToPreemptableResourceByPartition, Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates, Resource totalPreemptionAllowed, boolean readOnly) {
    RMContainer reservedContainer = node.getReservedContainer();
    Resource available = Resources.clone(node.getUnallocatedResource());
    Resource totalSelected = Resources.createResource(0);
    List<RMContainer> sortedRunningContainers = node.getCopiedListOfRunningContainers();
    List<RMContainer> selectedContainers = new ArrayList<>();
    Map<ContainerId, RMContainer> killableContainers = node.getKillableContainers();
    // Sort running container by launch time, we preferred to preempt recent
    // launched preempt container
    Collections.sort(sortedRunningContainers, new Comparator<RMContainer>() {

        @Override
        public int compare(RMContainer o1, RMContainer o2) {
            return -1 * o1.getContainerId().compareTo(o2.getContainerId());
        }
    });
    // First check: can we preempt containers to allocate the
    // reservedContainer?
    boolean canAllocateReservedContainer = false;
    // At least, we can get available + killable resources from this node
    Resource cur = Resources.add(available, node.getTotalKillableResources());
    String partition = node.getPartition();
    // Avoid preempt any container if required <= available + killable
    if (Resources.fitsIn(rc, cluster, reservedContainer.getReservedResource(), cur)) {
        return null;
    }
    // Extra cost of am container preemption
    float amPreemptionCost = 0f;
    for (RMContainer c : sortedRunningContainers) {
        String containerQueueName = c.getQueueName();
        // Skip container if it is already marked killable
        if (killableContainers.containsKey(c.getContainerId())) {
            continue;
        }
        // selected. Here for safety, avoid preempt AM container in any cases
        if (c.isAMContainer()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip selecting AM container on host=" + node.getNodeID() + " AM container=" + c.getContainerId());
            }
            continue;
        }
        // Can we preempt container c?
        // Check if we have quota to preempt this container
        boolean canPreempt = tryToPreemptFromQueue(cluster, containerQueueName, partition, queueToPreemptableResourceByPartition, c.getAllocatedResource(), totalPreemptionAllowed, readOnly);
        // If we can, add to selected container, and change resource accordingly.
        if (canPreempt) {
            if (!CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c, selectedCandidates)) {
                if (!readOnly) {
                    selectedContainers.add(c);
                }
                Resources.addTo(totalSelected, c.getAllocatedResource());
            }
            Resources.addTo(cur, c.getAllocatedResource());
            if (Resources.fitsIn(rc, cluster, reservedContainer.getReservedResource(), cur)) {
                canAllocateReservedContainer = true;
                break;
            }
        }
    }
    if (!canAllocateReservedContainer) {
        if (!readOnly) {
            // Revert queue preemption quotas
            for (RMContainer c : selectedContainers) {
                Resource res = getPreemptableResource(c.getQueueName(), partition, queueToPreemptableResourceByPartition);
                if (null == res) {
                    // Ignore such failures.
                    continue;
                }
                Resources.addTo(res, c.getAllocatedResource());
            }
        }
        return null;
    }
    float ratio = Resources.ratio(rc, totalSelected, reservedContainer.getReservedResource());
    // Compute preemption score
    NodeForPreemption nfp = new NodeForPreemption(ratio + amPreemptionCost, node, selectedContainers);
    return nfp;
}

Example 54 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class TempQueuePerPartition method offer.

// This function "accepts" all the resources it can (pending) and return
// the unused ones
Resource offer(Resource avail, ResourceCalculator rc, Resource clusterResource, boolean considersReservedResource) {
    Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(Resources.subtract(getMax(), idealAssigned), Resource.newInstance(0, 0));
    // accepted = min{avail,
    //               max - assigned,
    //               current + pending - assigned,
    //               # Make sure a queue will not get more than max of its
    //               # used/guaranteed, this is to make sure preemption won't
    //               # happen if all active queues are beyond their guaranteed
    //               # This is for leaf queue only.
    //               max(guaranteed, used) - assigned}
    // remain = avail - accepted
    Resource accepted = Resources.min(rc, clusterResource, absMaxCapIdealAssignedDelta, Resources.min(rc, clusterResource, avail, Resources.subtract(Resources.add(getUsed(), (considersReservedResource ? pending : pendingDeductReserved)), idealAssigned)));
    // from over-utilized leaf queue located at other hierarchies.
    if (null == children || children.isEmpty()) {
        Resource maxOfGuranteedAndUsedDeductAssigned = Resources.subtract(Resources.max(rc, clusterResource, getUsed(), getGuaranteed()), idealAssigned);
        maxOfGuranteedAndUsedDeductAssigned = Resources.max(rc, clusterResource, maxOfGuranteedAndUsedDeductAssigned, Resources.none());
        accepted = Resources.min(rc, clusterResource, accepted, maxOfGuranteedAndUsedDeductAssigned);
    }
    Resource remain = Resources.subtract(avail, accepted);
    Resources.addTo(idealAssigned, accepted);
    return remain;
}

Example 55 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class TempQueuePerPartition method assignPreemption.

public void assignPreemption(float scalingFactor, ResourceCalculator rc, Resource clusterResource) {
    Resource usedDeductKillable = Resources.subtract(getUsed(), killable);
    Resource totalResource = Resources.add(getUsed(), pending);
    // The minimum resource that we need to keep for a queue is:
    // max(idealAssigned, min(used + pending, guaranteed)).
    //
    // Doing this because when we calculate ideal allocation doesn't consider
    // reserved resource, ideal-allocation calculated could be less than
    // guaranteed and total. We should avoid preempt from a queue if it is
    // already
    // <= its guaranteed resource.
    Resource minimumQueueResource = Resources.max(rc, clusterResource, Resources.min(rc, clusterResource, totalResource, getGuaranteed()), idealAssigned);
    if (Resources.greaterThan(rc, clusterResource, usedDeductKillable, minimumQueueResource)) {
        toBePreempted = Resources.multiply(Resources.subtract(usedDeductKillable, minimumQueueResource), scalingFactor);
    } else {
        toBePreempted = Resources.none();
    }
}