Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 41 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class FSAppAttempt method allocate.

public RMContainer allocate(NodeType type, FSSchedulerNode node, SchedulerRequestKey schedulerKey, PendingAsk pendingAsk, Container reservedContainer) {
    RMContainer rmContainer;
    Container container;
    try {
        writeLock.lock();
        // Update allowed locality level
        NodeType allowed = allowedLocalityLevel.get(schedulerKey);
        if (allowed != null) {
            if (allowed.equals(NodeType.OFF_SWITCH) && (type.equals(NodeType.NODE_LOCAL) || type.equals(NodeType.RACK_LOCAL))) {
                this.resetAllowedLocalityLevel(schedulerKey, type);
            } else if (allowed.equals(NodeType.RACK_LOCAL) && type.equals(NodeType.NODE_LOCAL)) {
                this.resetAllowedLocalityLevel(schedulerKey, type);
            }
        }
        // request without locking the scheduler, hence we need to check
        if (getOutstandingAsksCount(schedulerKey) <= 0) {
            return null;
        }
        container = reservedContainer;
        if (container == null) {
            container = createContainer(node, pendingAsk.getPerAllocationResource(), schedulerKey);
        }
        // Create RMContainer
        rmContainer = new RMContainerImpl(container, schedulerKey, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
        ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
        // Add it to allContainers list.
        addToNewlyAllocatedContainers(node, rmContainer);
        liveContainers.put(container.getId(), rmContainer);
        // Update consumption and track allocations
        List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(type, node, schedulerKey, container);
        this.attemptResourceUsage.incUsed(container.getResource());
        // Update resource requests related to "request" and store in RMContainer
        ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
        // Inform the container
        rmContainer.handle(new RMContainerEvent(container.getId(), RMContainerEventType.START));
        if (LOG.isDebugEnabled()) {
            LOG.debug("allocate: applicationAttemptId=" + container.getId().getApplicationAttemptId() + " container=" + container.getId() + " host=" + container.getNodeId().getHost() + " type=" + type);
        }
        RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), container.getId(), container.getResource());
    } finally {
        writeLock.unlock();
    }
    return rmContainer;
}

Example 42 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class FSAppAttempt method getStarvedResourceRequests.

/**
   * Fetch a list of RRs corresponding to the extent the app is starved
   * (fairshare and minshare). This method considers the number of containers
   * in a RR and also only one locality-level (the first encountered
   * resourceName).
   *
   * @return list of {@link ResourceRequest}s corresponding to the amount of
   * starvation.
   */
List<ResourceRequest> getStarvedResourceRequests() {
    // List of RRs we build in this method to return
    List<ResourceRequest> ret = new ArrayList<>();
    // Track visited RRs to avoid the same RR at multiple locality levels
    VisitedResourceRequestTracker visitedRRs = new VisitedResourceRequestTracker(scheduler.getNodeTracker());
    // Start with current starvation and track the pending amount
    Resource pending = getStarvation();
    for (ResourceRequest rr : appSchedulingInfo.getAllResourceRequests()) {
        if (Resources.isNone(pending)) {
            // Found enough RRs to match the starvation
            break;
        }
        // See if we have already seen this RR
        if (!visitedRRs.visit(rr)) {
            continue;
        }
        // A RR can have multiple containers of a capability. We need to
        // compute the number of containers that fit in "pending".
        int numContainersThatFit = (int) Math.floor(Resources.ratio(scheduler.getResourceCalculator(), pending, rr.getCapability()));
        if (numContainersThatFit == 0) {
            // This RR's capability is too large to fit in pending
            continue;
        }
        // partial number of containers.
        if (numContainersThatFit < rr.getNumContainers()) {
            rr = ResourceRequest.newInstance(rr.getPriority(), rr.getResourceName(), rr.getCapability(), numContainersThatFit);
        }
        // Add the RR to return list and adjust "pending" accordingly
        ret.add(rr);
        Resources.subtractFromNonNegative(pending, Resources.multiply(rr.getCapability(), rr.getNumContainers()));
    }
    return ret;
}

Example 43 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class LocalitySchedulingPlacementSet method canAllocate.

@Override
public boolean canAllocate(NodeType type, SchedulerNode node) {
    try {
        readLock.lock();
        ResourceRequest r = resourceRequestMap.get(ResourceRequest.ANY);
        if (r == null || r.getNumContainers() <= 0) {
            return false;
        }
        if (type == NodeType.RACK_LOCAL || type == NodeType.NODE_LOCAL) {
            r = resourceRequestMap.get(node.getRackName());
            if (r == null || r.getNumContainers() <= 0) {
                return false;
            }
            if (type == NodeType.NODE_LOCAL) {
                r = resourceRequestMap.get(node.getNodeName());
                if (r == null || r.getNumContainers() <= 0) {
                    return false;
                }
            }
        }
        return true;
    } finally {
        readLock.unlock();
    }
}

Example 44 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class LocalitySchedulingPlacementSet method allocate.

@Override
public List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey, NodeType type, SchedulerNode node) {
    try {
        writeLock.lock();
        List<ResourceRequest> resourceRequests = new ArrayList<>();
        ResourceRequest request;
        if (type == NodeType.NODE_LOCAL) {
            request = resourceRequestMap.get(node.getNodeName());
        } else if (type == NodeType.RACK_LOCAL) {
            request = resourceRequestMap.get(node.getRackName());
        } else {
            request = resourceRequestMap.get(ResourceRequest.ANY);
        }
        if (type == NodeType.NODE_LOCAL) {
            allocateNodeLocal(schedulerKey, node, request, resourceRequests);
        } else if (type == NodeType.RACK_LOCAL) {
            allocateRackLocal(schedulerKey, node, request, resourceRequests);
        } else {
            allocateOffSwitch(schedulerKey, request, resourceRequests);
        }
        return resourceRequests;
    } finally {
        writeLock.unlock();
    }
}

Example 45 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class LocalitySchedulingPlacementSet method updateResourceRequests.

@Override
public ResourceRequestUpdateResult updateResourceRequests(Collection<ResourceRequest> requests, boolean recoverPreemptedRequestForAContainer) {
    try {
        this.writeLock.lock();
        ResourceRequestUpdateResult updateResult = null;
        // Update resource requests
        for (ResourceRequest request : requests) {
            String resourceName = request.getResourceName();
            // Update node labels if required
            updateNodeLabels(request);
            // Increment number of containers if recovering preempted resources
            ResourceRequest lastRequest = resourceRequestMap.get(resourceName);
            if (recoverPreemptedRequestForAContainer && lastRequest != null) {
                request.setNumContainers(lastRequest.getNumContainers() + 1);
            }
            // Update asks
            resourceRequestMap.put(resourceName, request);
            if (resourceName.equals(ResourceRequest.ANY)) {
                String partition = request.getNodeLabelExpression() == null ? RMNodeLabelsManager.NO_LABEL : request.getNodeLabelExpression();
                this.primaryRequestedPartition = partition;
                //update the applications requested labels set
                appSchedulingInfo.addRequestedPartition(partition);
                updateResult = new ResourceRequestUpdateResult(lastRequest, request);
            }
        }
        return updateResult;
    } finally {
        this.writeLock.unlock();
    }
}