Examples with PreemptionMessage org.apache.hadoop.yarn.api.records.PreemptionMessage used on opensource projects

Example 1 with PreemptionMessage

use of org.apache.hadoop.yarn.api.records.PreemptionMessage in project hadoop by apache.

the class TestCheckpointPreemptionPolicy method testStrictPreemptionContract.

@Test
public void testStrictPreemptionContract() {
    final Map<ContainerId, TaskAttemptId> containers = assignedContainers;
    AMPreemptionPolicy.Context mPctxt = new AMPreemptionPolicy.Context() {

        @Override
        public TaskAttemptId getTaskAttempt(ContainerId cId) {
            return containers.get(cId);
        }

        @Override
        public List<Container> getContainers(TaskType t) {
            List<Container> p = new ArrayList<Container>();
            for (Map.Entry<ContainerId, TaskAttemptId> ent : assignedContainers.entrySet()) {
                if (ent.getValue().getTaskId().getTaskType().equals(t)) {
                    p.add(Container.newInstance(ent.getKey(), null, null, contToResourceMap.get(ent.getKey()), Priority.newInstance(0), null));
                }
            }
            return p;
        }
    };
    PreemptionMessage pM = generatePreemptionMessage(preemptedContainers, contToResourceMap, Resource.newInstance(1024, 1), true);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(mActxt);
    policy.preempt(mPctxt, pM);
    for (ContainerId c : preemptedContainers) {
        TaskAttemptId t = assignedContainers.get(c);
        if (TaskType.MAP.equals(t.getTaskId().getTaskType())) {
            assert policy.isPreempted(t) == false;
        } else {
            assert policy.isPreempted(t);
        }
    }
}

Example 2 with PreemptionMessage

use of org.apache.hadoop.yarn.api.records.PreemptionMessage in project hadoop by apache.

the class TestCheckpointPreemptionPolicy method testPreemptionContract.

@Test
public void testPreemptionContract() {
    final Map<ContainerId, TaskAttemptId> containers = assignedContainers;
    AMPreemptionPolicy.Context mPctxt = new AMPreemptionPolicy.Context() {

        @Override
        public TaskAttemptId getTaskAttempt(ContainerId cId) {
            return containers.get(cId);
        }

        @Override
        public List<Container> getContainers(TaskType t) {
            List<Container> p = new ArrayList<Container>();
            for (Map.Entry<ContainerId, TaskAttemptId> ent : assignedContainers.entrySet()) {
                if (ent.getValue().getTaskId().getTaskType().equals(t)) {
                    p.add(Container.newInstance(ent.getKey(), null, null, contToResourceMap.get(ent.getKey()), Priority.newInstance(0), null));
                }
            }
            return p;
        }
    };
    PreemptionMessage pM = generatePreemptionMessage(preemptedContainers, contToResourceMap, Resource.newInstance(minAlloc, 1), false);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(mActxt);
    int supposedMemPreemption = (int) pM.getContract().getResourceRequest().get(0).getResourceRequest().getCapability().getMemorySize() * pM.getContract().getResourceRequest().get(0).getResourceRequest().getNumContainers();
    // first round of preemption
    policy.preempt(mPctxt, pM);
    List<TaskAttemptId> preempting = validatePreemption(pM, policy, supposedMemPreemption);
    // redundant message
    policy.preempt(mPctxt, pM);
    List<TaskAttemptId> preempting2 = validatePreemption(pM, policy, supposedMemPreemption);
    // check that nothing got added
    assert preempting2.equals(preempting);
    // simulate 2 task completions/successful preemption
    policy.handleCompletedContainer(preempting.get(0));
    policy.handleCompletedContainer(preempting.get(1));
    // remove from assignedContainers
    Iterator<Map.Entry<ContainerId, TaskAttemptId>> it = assignedContainers.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<ContainerId, TaskAttemptId> ent = it.next();
        if (ent.getValue().equals(preempting.get(0)) || ent.getValue().equals(preempting.get(1)))
            it.remove();
    }
    // one more message asking for preemption
    policy.preempt(mPctxt, pM);
    // triggers preemption of 2 more containers (i.e., the preemption set changes)
    List<TaskAttemptId> preempting3 = validatePreemption(pM, policy, supposedMemPreemption);
    assert preempting3.equals(preempting2) == false;
}

Example 3 with PreemptionMessage

use of org.apache.hadoop.yarn.api.records.PreemptionMessage in project hadoop by apache.

the class TestCheckpointPreemptionPolicy method generatePreemptionMessage.

private PreemptionMessage generatePreemptionMessage(Allocation allocation) {
    PreemptionMessage pMsg = null;
    // assemble strict preemption request
    if (allocation.getStrictContainerPreemptions() != null) {
        pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        StrictPreemptionContract pStrict = recordFactory.newRecordInstance(StrictPreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getStrictContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        pStrict.setContainers(pCont);
        pMsg.setStrictContract(pStrict);
    }
    // assemble negotiable preemption request
    if (allocation.getResourcePreemptions() != null && allocation.getResourcePreemptions().size() > 0 && allocation.getContainerPreemptions() != null && allocation.getContainerPreemptions().size() > 0) {
        if (pMsg == null) {
            pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        }
        PreemptionContract contract = recordFactory.newRecordInstance(PreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        List<PreemptionResourceRequest> pRes = new ArrayList<PreemptionResourceRequest>();
        for (ResourceRequest crr : allocation.getResourcePreemptions()) {
            PreemptionResourceRequest prr = recordFactory.newRecordInstance(PreemptionResourceRequest.class);
            prr.setResourceRequest(crr);
            pRes.add(prr);
        }
        contract.setContainers(pCont);
        contract.setResourceRequest(pRes);
        pMsg.setContract(contract);
    }
    return pMsg;
}

Example 4 with PreemptionMessage

use of org.apache.hadoop.yarn.api.records.PreemptionMessage in project hadoop by apache.

the class RMContainerAllocator method getResources.

@SuppressWarnings("unchecked")
private List<Container> getResources() throws Exception {
    applyConcurrentTaskLimits();
    // will be null the first time
    Resource headRoom = Resources.clone(getAvailableResources());
    AllocateResponse response;
    /*
     * If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS
     * milliseconds before aborting. During this interval, AM will still try
     * to contact the RM.
     */
    try {
        response = makeRemoteRequest();
        // Reset retry count if no exception occurred.
        retrystartTime = System.currentTimeMillis();
    } catch (ApplicationAttemptNotFoundException e) {
        // This can happen if the RM has been restarted. If it is in that state,
        // this application must clean itself up.
        eventHandler.handle(new JobEvent(this.getJob().getID(), JobEventType.JOB_AM_REBOOT));
        throw new RMContainerAllocationException("Resource Manager doesn't recognize AttemptId: " + this.getContext().getApplicationAttemptId(), e);
    } catch (ApplicationMasterNotRegisteredException e) {
        LOG.info("ApplicationMaster is out of sync with ResourceManager," + " hence resync and send outstanding requests.");
        // RM may have restarted, re-register with RM.
        lastResponseID = 0;
        register();
        addOutstandingRequestOnResync();
        return null;
    } catch (InvalidLabelResourceRequestException e) {
        // If Invalid label exception is received means the requested label doesnt
        // have access so killing job in this case.
        String diagMsg = "Requested node-label-expression is invalid: " + StringUtils.stringifyException(e);
        LOG.info(diagMsg);
        JobId jobId = this.getJob().getID();
        eventHandler.handle(new JobDiagnosticsUpdateEvent(jobId, diagMsg));
        eventHandler.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
        throw e;
    } catch (Exception e) {
        // re-trying until the retryInterval has expired.
        if (System.currentTimeMillis() - retrystartTime >= retryInterval) {
            LOG.error("Could not contact RM after " + retryInterval + " milliseconds.");
            eventHandler.handle(new JobEvent(this.getJob().getID(), JobEventType.JOB_AM_REBOOT));
            throw new RMContainerAllocationException("Could not contact RM after " + retryInterval + " milliseconds.");
        }
        // continue to attempt to contact the RM.
        throw e;
    }
    Resource newHeadRoom = getAvailableResources();
    List<Container> newContainers = response.getAllocatedContainers();
    // Setting NMTokens
    if (response.getNMTokens() != null) {
        for (NMToken nmToken : response.getNMTokens()) {
            NMTokenCache.setNMToken(nmToken.getNodeId().toString(), nmToken.getToken());
        }
    }
    // Setting AMRMToken
    if (response.getAMRMToken() != null) {
        updateAMRMToken(response.getAMRMToken());
    }
    List<ContainerStatus> finishedContainers = response.getCompletedContainersStatuses();
    // propagate preemption requests
    final PreemptionMessage preemptReq = response.getPreemptionMessage();
    if (preemptReq != null) {
        preemptionPolicy.preempt(new PreemptionContext(assignedRequests), preemptReq);
    }
    if (newContainers.size() + finishedContainers.size() > 0 || !headRoom.equals(newHeadRoom)) {
        //something changed
        recalculateReduceSchedule = true;
        if (LOG.isDebugEnabled() && !headRoom.equals(newHeadRoom)) {
            LOG.debug("headroom=" + newHeadRoom);
        }
    }
    if (LOG.isDebugEnabled()) {
        for (Container cont : newContainers) {
            LOG.debug("Received new Container :" + cont);
        }
    }
    //Called on each allocation. Will know about newly blacklisted/added hosts.
    computeIgnoreBlacklisting();
    handleUpdatedNodes(response);
    handleJobPriorityChange(response);
    // handle receiving the timeline collector address for this app
    String collectorAddr = response.getCollectorAddr();
    MRAppMaster.RunningAppContext appContext = (MRAppMaster.RunningAppContext) this.getContext();
    if (collectorAddr != null && !collectorAddr.isEmpty() && appContext.getTimelineV2Client() != null) {
        appContext.getTimelineV2Client().setTimelineServiceAddress(response.getCollectorAddr());
    }
    for (ContainerStatus cont : finishedContainers) {
        processFinishedContainer(cont);
    }
    return newContainers;
}

Example 5 with PreemptionMessage

use of org.apache.hadoop.yarn.api.records.PreemptionMessage in project hadoop by apache.

the class ApplicationMasterService method generatePreemptionMessage.

private PreemptionMessage generatePreemptionMessage(Allocation allocation) {
    PreemptionMessage pMsg = null;
    // assemble strict preemption request
    if (allocation.getStrictContainerPreemptions() != null) {
        pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        StrictPreemptionContract pStrict = recordFactory.newRecordInstance(StrictPreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getStrictContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        pStrict.setContainers(pCont);
        pMsg.setStrictContract(pStrict);
    }
    // assemble negotiable preemption request
    if (allocation.getResourcePreemptions() != null && allocation.getResourcePreemptions().size() > 0 && allocation.getContainerPreemptions() != null && allocation.getContainerPreemptions().size() > 0) {
        if (pMsg == null) {
            pMsg = recordFactory.newRecordInstance(PreemptionMessage.class);
        }
        PreemptionContract contract = recordFactory.newRecordInstance(PreemptionContract.class);
        Set<PreemptionContainer> pCont = new HashSet<PreemptionContainer>();
        for (ContainerId cId : allocation.getContainerPreemptions()) {
            PreemptionContainer pc = recordFactory.newRecordInstance(PreemptionContainer.class);
            pc.setId(cId);
            pCont.add(pc);
        }
        List<PreemptionResourceRequest> pRes = new ArrayList<PreemptionResourceRequest>();
        for (ResourceRequest crr : allocation.getResourcePreemptions()) {
            PreemptionResourceRequest prr = recordFactory.newRecordInstance(PreemptionResourceRequest.class);
            prr.setResourceRequest(crr);
            pRes.add(prr);
        }
        contract.setContainers(pCont);
        contract.setResourceRequest(pRes);
        pMsg.setContract(contract);
    }
    return pMsg;
}