Examples with PreemptionContract org.apache.hadoop.yarn.api.records.PreemptionContract used on opensource projects

Example 1 with PreemptionContract

use of org.apache.hadoop.yarn.api.records.PreemptionContract in project hadoop by apache.

the class TestCheckpointPreemptionPolicy method generatePreemptionMessage.

private PreemptionMessage generatePreemptionMessage(Allocation allocation) {
    PreemptionMessage pMsg = null;
    // assemble strict preemption request
    if (allocation.getStrictContainerPreemptions() != null) {
        pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        StrictPreemptionContract pStrict = recordFactory.newRecordInstance(StrictPreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getStrictContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        pStrict.setContainers(pCont);
        pMsg.setStrictContract(pStrict);
    }
    // assemble negotiable preemption request
    if (allocation.getResourcePreemptions() != null && allocation.getResourcePreemptions().size() > 0 && allocation.getContainerPreemptions() != null && allocation.getContainerPreemptions().size() > 0) {
        if (pMsg == null) {
            pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        }
        PreemptionContract contract = recordFactory.newRecordInstance(PreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        List<PreemptionResourceRequest> pRes = new ArrayList<PreemptionResourceRequest>();
        for (ResourceRequest crr : allocation.getResourcePreemptions()) {
            PreemptionResourceRequest prr = recordFactory.newRecordInstance(PreemptionResourceRequest.class);
            prr.setResourceRequest(crr);
            pRes.add(prr);
        }
        contract.setContainers(pCont);
        contract.setResourceRequest(pRes);
        pMsg.setContract(contract);
    }
    return pMsg;
}

Example 2 with PreemptionContract

use of org.apache.hadoop.yarn.api.records.PreemptionContract in project hadoop by apache.

the class CheckpointAMPreemptionPolicy method preempt.

@Override
public void preempt(Context ctxt, PreemptionMessage preemptionRequests) {
    if (preemptionRequests != null) {
        // handling non-negotiable preemption
        StrictPreemptionContract cStrict = preemptionRequests.getStrictContract();
        if (cStrict != null && cStrict.getContainers() != null && cStrict.getContainers().size() > 0) {
            LOG.info("strict preemption :" + preemptionRequests.getStrictContract().getContainers().size() + " containers to kill");
            // handle strict preemptions. These containers are non-negotiable
            for (PreemptionContainer c : preemptionRequests.getStrictContract().getContainers()) {
                ContainerId reqCont = c.getId();
                TaskAttemptId reqTask = ctxt.getTaskAttempt(reqCont);
                if (reqTask != null) {
                    // ignore requests for preempting containers running maps
                    if (org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE.equals(reqTask.getTaskId().getTaskType())) {
                        toBePreempted.add(reqTask);
                        LOG.info("preempting " + reqCont + " running task:" + reqTask);
                    } else {
                        LOG.info("NOT preempting " + reqCont + " running task:" + reqTask);
                    }
                }
            }
        }
        // handling negotiable preemption
        PreemptionContract cNegot = preemptionRequests.getContract();
        if (cNegot != null && cNegot.getResourceRequest() != null && cNegot.getResourceRequest().size() > 0 && cNegot.getContainers() != null && cNegot.getContainers().size() > 0) {
            LOG.info("negotiable preemption :" + preemptionRequests.getContract().getResourceRequest().size() + " resourceReq, " + preemptionRequests.getContract().getContainers().size() + " containers");
            // handle fungible preemption. Here we only look at the total amount of
            // resources to be preempted and pick enough of our containers to
            // satisfy that. We only support checkpointing for reducers for now.
            List<PreemptionResourceRequest> reqResources = preemptionRequests.getContract().getResourceRequest();
            // compute the total amount of pending preemptions (to be discounted
            // from current request)
            int pendingPreemptionRam = 0;
            int pendingPreemptionCores = 0;
            for (Resource r : pendingFlexiblePreemptions.values()) {
                pendingPreemptionRam += r.getMemorySize();
                pendingPreemptionCores += r.getVirtualCores();
            }
            // discount preemption request based on currently pending preemption
            for (PreemptionResourceRequest rr : reqResources) {
                ResourceRequest reqRsrc = rr.getResourceRequest();
                if (!ResourceRequest.ANY.equals(reqRsrc.getResourceName())) {
                    // For now, only respond to aggregate requests and ignore locality
                    continue;
                }
                LOG.info("ResourceRequest:" + reqRsrc);
                int reqCont = reqRsrc.getNumContainers();
                long reqMem = reqRsrc.getCapability().getMemorySize();
                long totalMemoryToRelease = reqCont * reqMem;
                int reqCores = reqRsrc.getCapability().getVirtualCores();
                int totalCoresToRelease = reqCont * reqCores;
                // remove
                if (pendingPreemptionRam > 0) {
                    // if goes negative we simply exit
                    totalMemoryToRelease -= pendingPreemptionRam;
                    // decrement pending resources if zero or negatve we will
                    // ignore it while processing next PreemptionResourceRequest
                    pendingPreemptionRam -= totalMemoryToRelease;
                }
                if (pendingPreemptionCores > 0) {
                    totalCoresToRelease -= pendingPreemptionCores;
                    pendingPreemptionCores -= totalCoresToRelease;
                }
                // reverse order of allocation (for now)
                List<Container> listOfCont = ctxt.getContainers(TaskType.REDUCE);
                Collections.sort(listOfCont, new Comparator<Container>() {

                    @Override
                    public int compare(final Container o1, final Container o2) {
                        return o2.getId().compareTo(o1.getId());
                    }
                });
                // preempt reducers first
                for (Container cont : listOfCont) {
                    if (totalMemoryToRelease <= 0 && totalCoresToRelease <= 0) {
                        break;
                    }
                    TaskAttemptId reduceId = ctxt.getTaskAttempt(cont.getId());
                    int cMem = (int) cont.getResource().getMemorySize();
                    int cCores = cont.getResource().getVirtualCores();
                    if (!toBePreempted.contains(reduceId)) {
                        totalMemoryToRelease -= cMem;
                        totalCoresToRelease -= cCores;
                        toBePreempted.add(reduceId);
                        pendingFlexiblePreemptions.put(reduceId, cont.getResource());
                    }
                    LOG.info("ResourceRequest:" + reqRsrc + " satisfied preempting " + reduceId);
                }
            // if map was preemptable we would do add them to toBePreempted here
            }
        }
    }
}

Example 3 with PreemptionContract

use of org.apache.hadoop.yarn.api.records.PreemptionContract in project hadoop by apache.

the class KillAMPreemptionPolicy method preempt.

@Override
public void preempt(Context ctxt, PreemptionMessage preemptionRequests) {
    // for both strict and negotiable preemption requests kill the
    // container
    StrictPreemptionContract strictContract = preemptionRequests.getStrictContract();
    if (strictContract != null) {
        for (PreemptionContainer c : strictContract.getContainers()) {
            killContainer(ctxt, c);
        }
    }
    PreemptionContract contract = preemptionRequests.getContract();
    if (contract != null) {
        for (PreemptionContainer c : contract.getContainers()) {
            killContainer(ctxt, c);
        }
    }
}

Example 4 with PreemptionContract

use of org.apache.hadoop.yarn.api.records.PreemptionContract in project hadoop by apache.

the class ApplicationMasterService method generatePreemptionMessage.

private PreemptionMessage generatePreemptionMessage(Allocation allocation) {
    PreemptionMessage pMsg = null;
    // assemble strict preemption request
    if (allocation.getStrictContainerPreemptions() != null) {
        pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        StrictPreemptionContract pStrict = recordFactory.newRecordInstance(StrictPreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getStrictContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        pStrict.setContainers(pCont);
        pMsg.setStrictContract(pStrict);
    }
    // assemble negotiable preemption request
    if (allocation.getResourcePreemptions() != null && allocation.getResourcePreemptions().size() > 0 && allocation.getContainerPreemptions() != null && allocation.getContainerPreemptions().size() > 0) {
        if (pMsg == null) {
            pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        }
        PreemptionContract contract = recordFactory.newRecordInstance(PreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        List<PreemptionResourceRequest> pRes = new ArrayList<PreemptionResourceRequest>();
        for (ResourceRequest crr : allocation.getResourcePreemptions()) {
            PreemptionResourceRequest prr = recordFactory.newRecordInstance(PreemptionResourceRequest.class);
            prr.setResourceRequest(crr);
            pRes.add(prr);
        }
        contract.setContainers(pCont);
        contract.setResourceRequest(pRes);
        pMsg.setContract(contract);
    }
    return pMsg;
}

Example 5 with PreemptionContract

use of org.apache.hadoop.yarn.api.records.PreemptionContract in project hadoop by apache.

the class TestKillAMPreemptionPolicy method getPreemptionMessage.

private PreemptionMessage getPreemptionMessage(boolean strictContract, boolean contract, final ContainerId container) {
    PreemptionMessage preemptionMessage = recordFactory.newRecordInstance(PreemptionMessage.class);
    Set<PreemptionContainer> cntrs = new HashSet<PreemptionContainer>();
    PreemptionContainer preemptContainer = recordFactory.newRecordInstance(PreemptionContainer.class);
    preemptContainer.setId(container);
    cntrs.add(preemptContainer);
    if (strictContract) {
        StrictPreemptionContract set = recordFactory.newRecordInstance(StrictPreemptionContract.class);
        set.setContainers(cntrs);
        preemptionMessage.setStrictContract(set);
    }
    if (contract) {
        PreemptionContract preemptContract = recordFactory.newRecordInstance(PreemptionContract.class);
        preemptContract.setContainers(cntrs);
        preemptionMessage.setContract(preemptContract);
    }
    return preemptionMessage;
}