Examples with Resource org.apache.hadoop.yarn.api.records.Resource used on opensource projects

Example 91 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class SchedulerApplicationAttempt method getRunningAggregateAppResourceUsage.

private AggregateAppResourceUsage getRunningAggregateAppResourceUsage() {
    long currentTimeMillis = System.currentTimeMillis();
    // recently.
    if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
        long memorySeconds = 0;
        long vcoreSeconds = 0;
        for (RMContainer rmContainer : this.liveContainers.values()) {
            long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
            Resource resource = rmContainer.getContainer().getResource();
            memorySeconds += resource.getMemorySize() * usedMillis / DateUtils.MILLIS_PER_SECOND;
            vcoreSeconds += resource.getVirtualCores() * usedMillis / DateUtils.MILLIS_PER_SECOND;
        }
        lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
        lastMemorySeconds = memorySeconds;
        lastVcoreSeconds = vcoreSeconds;
    }
    return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds);
}

Example 92 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class SchedulerApplicationAttempt method move.

public void move(Queue newQueue) {
    try {
        writeLock.lock();
        QueueMetrics oldMetrics = queue.getMetrics();
        QueueMetrics newMetrics = newQueue.getMetrics();
        String newQueueName = newQueue.getQueueName();
        String user = getUser();
        for (RMContainer liveContainer : liveContainers.values()) {
            Resource resource = liveContainer.getContainer().getResource();
            ((RMContainerImpl) liveContainer).setQueueName(newQueueName);
            oldMetrics.releaseResources(user, 1, resource);
            newMetrics.allocateResources(user, 1, resource, false);
        }
        for (Map<NodeId, RMContainer> map : reservedContainers.values()) {
            for (RMContainer reservedContainer : map.values()) {
                ((RMContainerImpl) reservedContainer).setQueueName(newQueueName);
                Resource resource = reservedContainer.getReservedResource();
                oldMetrics.unreserveResource(user, resource);
                newMetrics.reserveResource(user, resource);
            }
        }
        if (!isStopped) {
            appSchedulingInfo.move(newQueue);
        }
        this.queue = newQueue;
    } finally {
        writeLock.unlock();
    }
}

Example 93 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class FiCaSchedulerApp method commonCheckContainerAllocation.

private boolean commonCheckContainerAllocation(Resource cluster, ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation, SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer) {
    // Make sure node is not reserved by anyone else
    RMContainer reservedContainerOnNode = schedulerContainer.getSchedulerNode().getReservedContainer();
    if (reservedContainerOnNode != null) {
        RMContainer fromReservedContainer = allocation.getAllocateFromReservedContainer().getRmContainer();
        if (fromReservedContainer != reservedContainerOnNode) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Try to allocate from a non-existed reserved container");
            }
            return false;
        }
    }
    // Do we have enough space on this node?
    Resource availableResource = Resources.clone(schedulerContainer.getSchedulerNode().getUnallocatedResource());
    // of this node
    if (allocation.getToRelease() != null && !allocation.getToRelease().isEmpty()) {
        for (SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> releaseContainer : allocation.getToRelease()) {
            // not affect available resource of node) on the same node
            if (releaseContainer.getRmContainer().getState() != RMContainerState.RESERVED && releaseContainer.getSchedulerNode() == schedulerContainer.getSchedulerNode()) {
                Resources.addTo(availableResource, releaseContainer.getRmContainer().getAllocatedResource());
            }
        }
    }
    if (!Resources.fitsIn(rc, cluster, allocation.getAllocatedOrReservedResource(), availableResource)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Node doesn't have enough available resource, asked=" + allocation.getAllocatedOrReservedResource() + " available=" + availableResource);
        }
        return false;
    }
    return true;
}

Example 94 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class FiCaSchedulerApp method internalUnreserve.

private boolean internalUnreserve(FiCaSchedulerNode node, SchedulerRequestKey schedulerKey) {
    Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(schedulerKey);
    if (reservedContainers != null) {
        RMContainer reservedContainer = reservedContainers.remove(node.getNodeID());
        // as a consequence reservedcontainer might be null, adding NP-checks
        if (reservedContainer != null && reservedContainer.getContainer() != null && reservedContainer.getContainer().getResource() != null) {
            if (reservedContainers.isEmpty()) {
                this.reservedContainers.remove(schedulerKey);
            }
            // Reset the re-reservation count
            resetReReservations(schedulerKey);
            Resource resource = reservedContainer.getReservedResource();
            this.attemptResourceUsage.decReserved(node.getPartition(), resource);
            LOG.info("Application " + getApplicationId() + " unreserved " + " on node " + node + ", currently has " + reservedContainers.size() + " at priority " + schedulerKey.getPriority() + "; currentReservation " + this.attemptResourceUsage.getReserved() + " on node-label=" + node.getPartition());
            return true;
        }
    }
    return false;
}

Example 95 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class FiCaSchedulerApp method getAllocation.

/**
   * This method produces an Allocation that includes the current view
   * of the resources that will be allocated to and preempted from this
   * application.
   *
   * @param resourceCalculator resourceCalculator
   * @param clusterResource clusterResource
   * @param minimumAllocation minimumAllocation
   * @return an allocation
   */
public Allocation getAllocation(ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumAllocation) {
    try {
        writeLock.lock();
        Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(new HashSet<ContainerId>(containersToPreempt));
        containersToPreempt.clear();
        Resource tot = Resource.newInstance(0, 0);
        for (ContainerId c : currentContPreemption) {
            Resources.addTo(tot, liveContainers.get(c).getContainer().getResource());
        }
        int numCont = (int) Math.ceil(Resources.divide(rc, clusterResource, tot, minimumAllocation));
        ResourceRequest rr = ResourceRequest.newBuilder().priority(Priority.UNDEFINED).resourceName(ResourceRequest.ANY).capability(minimumAllocation).numContainers(numCont).build();
        List<Container> newlyAllocatedContainers = pullNewlyAllocatedContainers();
        List<Container> newlyIncreasedContainers = pullNewlyIncreasedContainers();
        List<Container> newlyDecreasedContainers = pullNewlyDecreasedContainers();
        List<Container> newlyPromotedContainers = pullNewlyPromotedContainers();
        List<Container> newlyDemotedContainers = pullNewlyDemotedContainers();
        List<NMToken> updatedNMTokens = pullUpdatedNMTokens();
        Resource headroom = getHeadroom();
        setApplicationHeadroomForMetrics(headroom);
        return new Allocation(newlyAllocatedContainers, headroom, null, currentContPreemption, Collections.singletonList(rr), updatedNMTokens, newlyIncreasedContainers, newlyDecreasedContainers, newlyPromotedContainers, newlyDemotedContainers);
    } finally {
        writeLock.unlock();
    }
}