Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 46 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class LocalitySchedulingPlacementSet method allocate.

@Override
public List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey, NodeType type, SchedulerNode node) {
    try {
        writeLock.lock();
        List<ResourceRequest> resourceRequests = new ArrayList<>();
        ResourceRequest request;
        if (type == NodeType.NODE_LOCAL) {
            request = resourceRequestMap.get(node.getNodeName());
        } else if (type == NodeType.RACK_LOCAL) {
            request = resourceRequestMap.get(node.getRackName());
        } else {
            request = resourceRequestMap.get(ResourceRequest.ANY);
        }
        if (type == NodeType.NODE_LOCAL) {
            allocateNodeLocal(schedulerKey, node, request, resourceRequests);
        } else if (type == NodeType.RACK_LOCAL) {
            allocateRackLocal(schedulerKey, node, request, resourceRequests);
        } else {
            allocateOffSwitch(schedulerKey, request, resourceRequests);
        }
        return resourceRequests;
    } finally {
        writeLock.unlock();
    }
}

Example 47 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class LocalitySchedulingPlacementSet method updateResourceRequests.

@Override
public ResourceRequestUpdateResult updateResourceRequests(Collection<ResourceRequest> requests, boolean recoverPreemptedRequestForAContainer) {
    try {
        this.writeLock.lock();
        ResourceRequestUpdateResult updateResult = null;
        // Update resource requests
        for (ResourceRequest request : requests) {
            String resourceName = request.getResourceName();
            // Update node labels if required
            updateNodeLabels(request);
            // Increment number of containers if recovering preempted resources
            ResourceRequest lastRequest = resourceRequestMap.get(resourceName);
            if (recoverPreemptedRequestForAContainer && lastRequest != null) {
                request.setNumContainers(lastRequest.getNumContainers() + 1);
            }
            // Update asks
            resourceRequestMap.put(resourceName, request);
            if (resourceName.equals(ResourceRequest.ANY)) {
                String partition = request.getNodeLabelExpression() == null ? RMNodeLabelsManager.NO_LABEL : request.getNodeLabelExpression();
                this.primaryRequestedPartition = partition;
                //update the applications requested labels set
                appSchedulingInfo.addRequestedPartition(partition);
                updateResult = new ResourceRequestUpdateResult(lastRequest, request);
            }
        }
        return updateResult;
    } finally {
        this.writeLock.unlock();
    }
}

Example 48 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class LocalitySchedulingPlacementSet method allocateRackLocal.

/**
   * The {@link ResourceScheduler} is allocating data-local resources to the
   * application.
   */
private void allocateRackLocal(SchedulerRequestKey schedulerKey, SchedulerNode node, ResourceRequest rackLocalRequest, List<ResourceRequest> resourceRequests) {
    // Update future requirements
    decResourceRequest(node.getRackName(), rackLocalRequest);
    ResourceRequest offRackRequest = resourceRequestMap.get(ResourceRequest.ANY);
    decrementOutstanding(schedulerKey, offRackRequest);
    // Update cloned RackLocal and OffRack requests for recovery
    resourceRequests.add(cloneResourceRequest(rackLocalRequest));
    resourceRequests.add(cloneResourceRequest(offRackRequest));
}

Example 49 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class LocalitySchedulingPlacementSet method getPendingAsk.

@Override
public PendingAsk getPendingAsk(String resourceName) {
    try {
        readLock.lock();
        ResourceRequest request = getResourceRequest(resourceName);
        if (null == request) {
            return PendingAsk.ZERO;
        } else {
            return new PendingAsk(request.getCapability(), request.getNumContainers());
        }
    } finally {
        readLock.unlock();
    }
}

Example 50 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class AMRMClientImpl method allocate.

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();
    Map<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> oldChange = new HashMap<>();
    try {
        synchronized (this) {
            askList = cloneAsks();
            // Save the current change for recovery
            oldChange.putAll(change);
            List<UpdateContainerRequest> updateList = createUpdateList();
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();
            change.clear();
            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove);
            allocateRequest = AllocateRequest.newBuilder().responseId(lastResponseId).progress(progressIndicator).askList(askList).resourceBlacklistRequest(blacklistRequest).releaseList(releaseList).updateRequests(updateList).build();
            // clear blacklistAdditions and blacklistRemovals before
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }
        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (RemoteRequestsTable remoteRequestsTable : remoteRequests.values()) {
                    @SuppressWarnings("unchecked") Iterator<ResourceRequestInfo<T>> reqIter = remoteRequestsTable.iterator();
                    while (reqIter.hasNext()) {
                        addResourceRequestToAsk(reqIter.next().remoteRequest);
                    }
                }
                change.putAll(this.pendingChange);
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }
        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
            if (!pendingChange.isEmpty()) {
                List<ContainerStatus> completed = allocateResponse.getCompletedContainersStatuses();
                List<UpdatedContainer> changed = new ArrayList<>();
                changed.addAll(allocateResponse.getUpdatedContainers());
                // containers
                for (ContainerStatus status : completed) {
                    ContainerId containerId = status.getContainerId();
                    pendingChange.remove(containerId);
                }
                // remove all pending change requests that have been satisfied
                if (!changed.isEmpty()) {
                    removePendingChangeRequests(changed);
                }
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }
                // that do not exist in the current change map:
                for (Map.Entry<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> entry : oldChange.entrySet()) {
                    ContainerId oldContainerId = entry.getKey();
                    Container oldContainer = entry.getValue().getKey();
                    UpdateContainerRequest oldupdate = entry.getValue().getValue();
                    if (change.get(oldContainerId) == null) {
                        change.put(oldContainerId, new SimpleEntry<>(oldContainer, oldupdate));
                    }
                }
                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}