Examples with AllocateResponse org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse used on opensource projects

Example 66 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class ApplicationMasterService method registerAppAttempt.

public void registerAppAttempt(ApplicationAttemptId attemptId) {
    AllocateResponse response = recordFactory.newRecordInstance(AllocateResponse.class);
    // set response id to -1 before application master for the following
    // attemptID get registered
    response.setResponseId(-1);
    LOG.info("Registering app attempt : " + attemptId);
    responseMap.put(attemptId, new AllocateResponseLock(response));
    rmContext.getNMTokenSecretManager().registerApplicationAttempt(attemptId);
}

Example 67 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class ApplicationMasterService method allocate.

@Override
public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException {
    AMRMTokenIdentifier amrmTokenIdentifier = YarnServerSecurityUtils.authorizeRequest();
    ApplicationAttemptId appAttemptId = amrmTokenIdentifier.getApplicationAttemptId();
    this.amLivelinessMonitor.receivedPing(appAttemptId);
    /* check if its in cache */
    AllocateResponseLock lock = responseMap.get(appAttemptId);
    if (lock == null) {
        String message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache.";
        LOG.error(message);
        throw new ApplicationAttemptNotFoundException(message);
    }
    synchronized (lock) {
        AllocateResponse lastResponse = lock.getAllocateResponse();
        if (!hasApplicationMasterRegistered(appAttemptId)) {
            String message = "AM is not registered for known application attempt: " + appAttemptId + " or RM had restarted after AM registered. " + " AM should re-register.";
            throw new ApplicationMasterNotRegisteredException(message);
        }
        if ((request.getResponseId() + 1) == lastResponse.getResponseId()) {
            /* old heartbeat */
            return lastResponse;
        } else if (request.getResponseId() + 1 < lastResponse.getResponseId()) {
            String message = "Invalid responseId in AllocateRequest from application attempt: " + appAttemptId + ", expect responseId to be " + (lastResponse.getResponseId() + 1);
            throw new InvalidApplicationMasterRequestException(message);
        }
        AllocateResponse response = recordFactory.newRecordInstance(AllocateResponse.class);
        allocateInternal(amrmTokenIdentifier.getApplicationAttemptId(), request, response);
        // update AMRMToken if the token is rolled-up
        MasterKeyData nextMasterKey = this.rmContext.getAMRMTokenSecretManager().getNextMasterKeyData();
        if (nextMasterKey != null && nextMasterKey.getMasterKey().getKeyId() != amrmTokenIdentifier.getKeyId()) {
            RMApp app = this.rmContext.getRMApps().get(appAttemptId.getApplicationId());
            RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
            RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl) appAttempt;
            Token<AMRMTokenIdentifier> amrmToken = appAttempt.getAMRMToken();
            if (nextMasterKey.getMasterKey().getKeyId() != appAttemptImpl.getAMRMTokenKeyId()) {
                LOG.info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: " + appAttemptId.getApplicationId());
                amrmToken = rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(appAttemptId);
                appAttemptImpl.setAMRMToken(amrmToken);
            }
            response.setAMRMToken(org.apache.hadoop.yarn.api.records.Token.newInstance(amrmToken.getIdentifier(), amrmToken.getKind().toString(), amrmToken.getPassword(), amrmToken.getService().toString()));
        }
        /*
       * As we are updating the response inside the lock object so we don't
       * need to worry about unregister call occurring in between (which
       * removes the lock object).
       */
        response.setResponseId(lastResponse.getResponseId() + 1);
        lock.setAllocateResponse(response);
        return response;
    }
}

Example 68 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class OpportunisticContainerAllocatorAMService method allocateForDistributedScheduling.

@Override
public DistributedSchedulingAllocateResponse allocateForDistributedScheduling(DistributedSchedulingAllocateRequest request) throws YarnException, IOException {
    List<Container> distAllocContainers = request.getAllocatedContainers();
    handleNewContainers(distAllocContainers, true);
    AllocateResponse response = allocate(request.getAllocateRequest());
    DistributedSchedulingAllocateResponse dsResp = recordFactory.newRecordInstance(DistributedSchedulingAllocateResponse.class);
    dsResp.setAllocateResponse(response);
    dsResp.setNodesForScheduling(getLeastLoadedNodes());
    return dsResp;
}

Example 69 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class LocalContainerAllocator method heartbeat.

@SuppressWarnings("unchecked")
@Override
protected synchronized void heartbeat() throws Exception {
    AllocateRequest allocateRequest = AllocateRequest.newInstance(this.lastResponseID, super.getApplicationProgress(), new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), null);
    AllocateResponse allocateResponse = null;
    try {
        allocateResponse = scheduler.allocate(allocateRequest);
        // Reset retry count if no exception occurred.
        retrystartTime = System.currentTimeMillis();
    } catch (ApplicationAttemptNotFoundException e) {
        LOG.info("Event from RM: shutting down Application Master");
        // This can happen if the RM has been restarted. If it is in that state,
        // this application must clean itself up.
        eventHandler.handle(new JobEvent(this.getJob().getID(), JobEventType.JOB_AM_REBOOT));
        throw new YarnRuntimeException("Resource Manager doesn't recognize AttemptId: " + this.getContext().getApplicationID(), e);
    } catch (ApplicationMasterNotRegisteredException e) {
        LOG.info("ApplicationMaster is out of sync with ResourceManager," + " hence resync and send outstanding requests.");
        this.lastResponseID = 0;
        register();
    } catch (Exception e) {
        // re-trying until the retryInterval has expired.
        if (System.currentTimeMillis() - retrystartTime >= retryInterval) {
            LOG.error("Could not contact RM after " + retryInterval + " milliseconds.");
            eventHandler.handle(new JobEvent(this.getJob().getID(), JobEventType.INTERNAL_ERROR));
            throw new YarnRuntimeException("Could not contact RM after " + retryInterval + " milliseconds.");
        }
        // continue to attempt to contact the RM.
        throw e;
    }
    if (allocateResponse != null) {
        this.lastResponseID = allocateResponse.getResponseId();
        Token token = allocateResponse.getAMRMToken();
        if (token != null) {
            updateAMRMToken(token);
        }
        Priority priorityFromResponse = Priority.newInstance(allocateResponse.getApplicationPriority().getPriority());
        // Update the job priority to Job directly.
        getJob().setJobPriority(priorityFromResponse);
    }
}

Example 70 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class MRAppBenchmark method benchmark1.

@Test
public void benchmark1() throws Exception {
    // Adjust for benchmarking. Start with thousands.
    int maps = 100;
    int reduces = 0;
    System.out.println("Running benchmark with maps:" + maps + " reduces:" + reduces);
    run(new MRApp(maps, reduces, true, this.getClass().getName(), true) {

        @Override
        protected ContainerAllocator createContainerAllocator(ClientService clientService, AppContext context) {
            AMPreemptionPolicy policy = new NoopAMPreemptionPolicy();
            return new RMContainerAllocator(clientService, context, policy) {

                @Override
                protected ApplicationMasterProtocol createSchedulerProxy() {
                    return new ApplicationMasterProtocol() {

                        @Override
                        public RegisterApplicationMasterResponse registerApplicationMaster(RegisterApplicationMasterRequest request) throws IOException {
                            RegisterApplicationMasterResponse response = Records.newRecord(RegisterApplicationMasterResponse.class);
                            response.setMaximumResourceCapability(Resource.newInstance(10240, 1));
                            return response;
                        }

                        @Override
                        public FinishApplicationMasterResponse finishApplicationMaster(FinishApplicationMasterRequest request) throws IOException {
                            FinishApplicationMasterResponse response = Records.newRecord(FinishApplicationMasterResponse.class);
                            return response;
                        }

                        @Override
                        public AllocateResponse allocate(AllocateRequest request) throws IOException {
                            AllocateResponse response = Records.newRecord(AllocateResponse.class);
                            List<ResourceRequest> askList = request.getAskList();
                            List<Container> containers = new ArrayList<Container>();
                            for (ResourceRequest req : askList) {
                                if (!ResourceRequest.isAnyLocation(req.getResourceName())) {
                                    continue;
                                }
                                int numContainers = req.getNumContainers();
                                for (int i = 0; i < numContainers; i++) {
                                    ContainerId containerId = ContainerId.newContainerId(getContext().getApplicationAttemptId(), request.getResponseId() + i);
                                    containers.add(Container.newInstance(containerId, NodeId.newInstance("host" + containerId.getContainerId(), 2345), "host" + containerId.getContainerId() + ":5678", req.getCapability(), req.getPriority(), null));
                                }
                            }
                            response.setAllocatedContainers(containers);
                            response.setResponseId(request.getResponseId() + 1);
                            response.setNumClusterNodes(350);
                            return response;
                        }
                    };
                }
            };
        }
    });
}