Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 66 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestCheckpointPreemptionPolicy method generatePreemptionMessage.

private PreemptionMessage generatePreemptionMessage(Allocation allocation) {
    PreemptionMessage pMsg = null;
    // assemble strict preemption request
    if (allocation.getStrictContainerPreemptions() != null) {
        pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        StrictPreemptionContract pStrict = recordFactory.newRecordInstance(StrictPreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getStrictContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        pStrict.setContainers(pCont);
        pMsg.setStrictContract(pStrict);
    }
    // assemble negotiable preemption request
    if (allocation.getResourcePreemptions() != null && allocation.getResourcePreemptions().size() > 0 && allocation.getContainerPreemptions() != null && allocation.getContainerPreemptions().size() > 0) {
        if (pMsg == null) {
            pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        }
        PreemptionContract contract = recordFactory.newRecordInstance(PreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        List<PreemptionResourceRequest> pRes = new ArrayList<PreemptionResourceRequest>();
        for (ResourceRequest crr : allocation.getResourcePreemptions()) {
            PreemptionResourceRequest prr = recordFactory.newRecordInstance(PreemptionResourceRequest.class);
            prr.setResourceRequest(crr);
            pRes.add(prr);
        }
        contract.setContainers(pCont);
        contract.setResourceRequest(pRes);
        pMsg.setContract(contract);
    }
    return pMsg;
}

Example 67 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class RMContainerRequestor method addResourceRequest.

private void addResourceRequest(Priority priority, String resourceName, Resource capability, String nodeLabelExpression, ExecutionType executionType) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);
    if (remoteRequests == null) {
        remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
        this.remoteRequestsTable.put(priority, remoteRequests);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Added priority=" + priority);
        }
    }
    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        reqMap = new HashMap<Resource, ResourceRequest>();
        remoteRequests.put(resourceName, reqMap);
    }
    ResourceRequest remoteRequest = reqMap.get(capability);
    if (remoteRequest == null) {
        remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
        remoteRequest.setPriority(priority);
        remoteRequest.setResourceName(resourceName);
        remoteRequest.setCapability(capability);
        remoteRequest.setNumContainers(0);
        remoteRequest.setNodeLabelExpression(nodeLabelExpression);
        remoteRequest.setExecutionTypeRequest(ExecutionTypeRequest.newInstance(executionType, true));
        reqMap.put(capability, remoteRequest);
    }
    remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);
    // Note this down for next interaction with ResourceManager
    addResourceRequestToAsk(remoteRequest);
    if (LOG.isDebugEnabled()) {
        LOG.debug("addResourceRequest:" + " applicationId=" + applicationId.getId() + " priority=" + priority.getPriority() + " resourceName=" + resourceName + " numContainers=" + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

Example 68 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class RMContainerRequestor method makeRemoteRequest.

protected AllocateResponse makeRemoteRequest() throws YarnException, IOException {
    applyRequestLimits();
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions), new ArrayList<String>(blacklistRemovals));
    AllocateRequest allocateRequest = AllocateRequest.newInstance(lastResponseID, super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>(release), blacklistRequest);
    AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
    lastResponseID = allocateResponse.getResponseId();
    availableResources = allocateResponse.getAvailableResources();
    lastClusterNmCount = clusterNmCount;
    clusterNmCount = allocateResponse.getNumClusterNodes();
    int numCompletedContainers = allocateResponse.getCompletedContainersStatuses().size();
    if (ask.size() > 0 || release.size() > 0) {
        LOG.info("getResources() for " + applicationId + ":" + " ask=" + ask.size() + " release= " + release.size() + " newContainers=" + allocateResponse.getAllocatedContainers().size() + " finishedContainers=" + numCompletedContainers + " resourcelimit=" + availableResources + " knownNMs=" + clusterNmCount);
    }
    ask.clear();
    release.clear();
    if (numCompletedContainers > 0) {
        // re-send limited requests when a container completes to trigger asking
        // for more containers
        requestLimitsToUpdate.addAll(requestLimits.keySet());
    }
    if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
        LOG.info("Update the blacklist for " + applicationId + ": blacklistAdditions=" + blacklistAdditions.size() + " blacklistRemovals=" + blacklistRemovals.size());
    }
    blacklistAdditions.clear();
    blacklistRemovals.clear();
    return allocateResponse;
}

Example 69 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class RMContainerRequestor method applyRequestLimits.

private void applyRequestLimits() {
    Iterator<ResourceRequest> iter = requestLimits.values().iterator();
    while (iter.hasNext()) {
        ResourceRequest reqLimit = iter.next();
        int limit = reqLimit.getNumContainers();
        Map<String, Map<Resource, ResourceRequest>> remoteRequests = remoteRequestsTable.get(reqLimit.getPriority());
        Map<Resource, ResourceRequest> reqMap = (remoteRequests != null) ? remoteRequests.get(ResourceRequest.ANY) : null;
        ResourceRequest req = (reqMap != null) ? reqMap.get(reqLimit.getCapability()) : null;
        if (req == null) {
            continue;
        }
        // update an existing ask or send a new one if updating
        if (ask.remove(req) || requestLimitsToUpdate.contains(req)) {
            ResourceRequest newReq = req.getNumContainers() > limit ? reqLimit : req;
            ask.add(newReq);
            LOG.info("Applying ask limit of " + newReq.getNumContainers() + " for priority:" + reqLimit.getPriority() + " and capability:" + reqLimit.getCapability());
        }
        if (limit == Integer.MAX_VALUE) {
            iter.remove();
        }
    }
    requestLimitsToUpdate.clear();
}

Example 70 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class CheckpointAMPreemptionPolicy method preempt.

@Override
public void preempt(Context ctxt, PreemptionMessage preemptionRequests) {
    if (preemptionRequests != null) {
        // handling non-negotiable preemption
        StrictPreemptionContract cStrict = preemptionRequests.getStrictContract();
        if (cStrict != null && cStrict.getContainers() != null && cStrict.getContainers().size() > 0) {
            LOG.info("strict preemption :" + preemptionRequests.getStrictContract().getContainers().size() + " containers to kill");
            // handle strict preemptions. These containers are non-negotiable
            for (PreemptionContainer c : preemptionRequests.getStrictContract().getContainers()) {
                ContainerId reqCont = c.getId();
                TaskAttemptId reqTask = ctxt.getTaskAttempt(reqCont);
                if (reqTask != null) {
                    // ignore requests for preempting containers running maps
                    if (org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE.equals(reqTask.getTaskId().getTaskType())) {
                        toBePreempted.add(reqTask);
                        LOG.info("preempting " + reqCont + " running task:" + reqTask);
                    } else {
                        LOG.info("NOT preempting " + reqCont + " running task:" + reqTask);
                    }
                }
            }
        }
        // handling negotiable preemption
        PreemptionContract cNegot = preemptionRequests.getContract();
        if (cNegot != null && cNegot.getResourceRequest() != null && cNegot.getResourceRequest().size() > 0 && cNegot.getContainers() != null && cNegot.getContainers().size() > 0) {
            LOG.info("negotiable preemption :" + preemptionRequests.getContract().getResourceRequest().size() + " resourceReq, " + preemptionRequests.getContract().getContainers().size() + " containers");
            // handle fungible preemption. Here we only look at the total amount of
            // resources to be preempted and pick enough of our containers to
            // satisfy that. We only support checkpointing for reducers for now.
            List<PreemptionResourceRequest> reqResources = preemptionRequests.getContract().getResourceRequest();
            // compute the total amount of pending preemptions (to be discounted
            // from current request)
            int pendingPreemptionRam = 0;
            int pendingPreemptionCores = 0;
            for (Resource r : pendingFlexiblePreemptions.values()) {
                pendingPreemptionRam += r.getMemorySize();
                pendingPreemptionCores += r.getVirtualCores();
            }
            // discount preemption request based on currently pending preemption
            for (PreemptionResourceRequest rr : reqResources) {
                ResourceRequest reqRsrc = rr.getResourceRequest();
                if (!ResourceRequest.ANY.equals(reqRsrc.getResourceName())) {
                    // For now, only respond to aggregate requests and ignore locality
                    continue;
                }
                LOG.info("ResourceRequest:" + reqRsrc);
                int reqCont = reqRsrc.getNumContainers();
                long reqMem = reqRsrc.getCapability().getMemorySize();
                long totalMemoryToRelease = reqCont * reqMem;
                int reqCores = reqRsrc.getCapability().getVirtualCores();
                int totalCoresToRelease = reqCont * reqCores;
                // remove
                if (pendingPreemptionRam > 0) {
                    // if goes negative we simply exit
                    totalMemoryToRelease -= pendingPreemptionRam;
                    // decrement pending resources if zero or negatve we will
                    // ignore it while processing next PreemptionResourceRequest
                    pendingPreemptionRam -= totalMemoryToRelease;
                }
                if (pendingPreemptionCores > 0) {
                    totalCoresToRelease -= pendingPreemptionCores;
                    pendingPreemptionCores -= totalCoresToRelease;
                }
                // reverse order of allocation (for now)
                List<Container> listOfCont = ctxt.getContainers(TaskType.REDUCE);
                Collections.sort(listOfCont, new Comparator<Container>() {

                    @Override
                    public int compare(final Container o1, final Container o2) {
                        return o2.getId().compareTo(o1.getId());
                    }
                });
                // preempt reducers first
                for (Container cont : listOfCont) {
                    if (totalMemoryToRelease <= 0 && totalCoresToRelease <= 0) {
                        break;
                    }
                    TaskAttemptId reduceId = ctxt.getTaskAttempt(cont.getId());
                    int cMem = (int) cont.getResource().getMemorySize();
                    int cCores = cont.getResource().getVirtualCores();
                    if (!toBePreempted.contains(reduceId)) {
                        totalMemoryToRelease -= cMem;
                        totalCoresToRelease -= cCores;
                        toBePreempted.add(reduceId);
                        pendingFlexiblePreemptions.put(reduceId, cont.getResource());
                    }
                    LOG.info("ResourceRequest:" + reqRsrc + " satisfied preempting " + reduceId);
                }
            // if map was preemptable we would do add them to toBePreempted here
            }
        }
    }
}