Examples with AllocateResponse org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse used on opensource projects

Example 31 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestDistributedScheduling method testAMRMClient.

/**
   * Validates if AMRMClient can be used with Distributed Scheduling turned on.
   *
   * @throws Exception
   */
@Test(timeout = 120000)
@SuppressWarnings("unchecked")
public void testAMRMClient() throws Exception {
    AMRMClientImpl<AMRMClient.ContainerRequest> amClient = null;
    try {
        Priority priority = Priority.newInstance(1);
        Priority priority2 = Priority.newInstance(2);
        Resource capability = Resource.newInstance(1024, 1);
        List<NodeReport> nodeReports = rmClient.getNodeReports(NodeState.RUNNING);
        String node = nodeReports.get(0).getNodeId().getHost();
        String rack = nodeReports.get(0).getRackName();
        String[] nodes = new String[] { node };
        String[] racks = new String[] { rack };
        // start am rm client
        amClient = new AMRMClientImpl(client);
        amClient.init(yarnConf);
        amClient.start();
        amClient.registerApplicationMaster(NetUtils.getHostname(), 1024, "");
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        RemoteRequestsTable<ContainerRequest> remoteRequestsTable = amClient.getTable(0);
        int containersRequestedNode = remoteRequestsTable.get(priority, node, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int containersRequestedRack = remoteRequestsTable.get(priority, rack, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int containersRequestedAny = remoteRequestsTable.get(priority, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int oppContainersRequestedAny = remoteRequestsTable.get(priority2, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC, capability).remoteRequest.getNumContainers();
        assertEquals(2, containersRequestedNode);
        assertEquals(2, containersRequestedRack);
        assertEquals(2, containersRequestedAny);
        assertEquals(1, oppContainersRequestedAny);
        assertEquals(4, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        // RM should allocate container within 2 calls to allocate()
        int allocatedContainerCount = 0;
        int iterationsLeft = 10;
        Set<ContainerId> releases = new TreeSet<>();
        amClient.getNMTokenCache().clearCache();
        Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
        HashMap<String, Token> receivedNMTokens = new HashMap<>();
        while (allocatedContainerCount < (containersRequestedAny + oppContainersRequestedAny) && iterationsLeft-- > 0) {
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            assertEquals(0, amClient.ask.size());
            assertEquals(0, amClient.release.size());
            allocatedContainerCount += allocResponse.getAllocatedContainers().size();
            for (Container container : allocResponse.getAllocatedContainers()) {
                ContainerId rejectContainerId = container.getId();
                releases.add(rejectContainerId);
            }
            for (NMToken token : allocResponse.getNMTokens()) {
                String nodeID = token.getNodeId().toString();
                receivedNMTokens.put(nodeID, token.getToken());
            }
            if (allocatedContainerCount < containersRequestedAny) {
                // sleep to let NM's heartbeat to RM and trigger allocations
                sleep(100);
            }
        }
        assertEquals(allocatedContainerCount, containersRequestedAny + oppContainersRequestedAny);
        for (ContainerId rejectContainerId : releases) {
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        assertEquals(3, amClient.release.size());
        assertEquals(0, amClient.ask.size());
        // need to tell the AMRMClient that we dont need these resources anymore
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        assertEquals(4, amClient.ask.size());
        // test RPC exception handling
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        final AMRMClient amc = amClient;
        ApplicationMasterProtocol realRM = amClient.rmClient;
        try {
            ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
            when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(new Answer<AllocateResponse>() {

                public AllocateResponse answer(InvocationOnMock invocation) throws Exception {
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
                    throw new Exception();
                }
            });
            amClient.rmClient = mockRM;
            amClient.allocate(0.1f);
        } catch (Exception ioe) {
        } finally {
            amClient.rmClient = realRM;
        }
        assertEquals(3, amClient.release.size());
        assertEquals(6, amClient.ask.size());
        iterationsLeft = 3;
        // do a few iterations to ensure RM is not going send new containers
        while (iterationsLeft-- > 0) {
            // inform RM of rejection
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            // RM did not send new containers because AM does not need any
            assertEquals(0, allocResponse.getAllocatedContainers().size());
            if (allocResponse.getCompletedContainersStatuses().size() > 0) {
                for (ContainerStatus cStatus : allocResponse.getCompletedContainersStatuses()) {
                    if (releases.contains(cStatus.getContainerId())) {
                        assertEquals(cStatus.getState(), ContainerState.COMPLETE);
                        assertEquals(-100, cStatus.getExitStatus());
                        releases.remove(cStatus.getContainerId());
                    }
                }
            }
            if (iterationsLeft > 0) {
                // sleep to make sure NM's heartbeat
                sleep(100);
            }
        }
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == Service.STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 32 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class ApplicationMasterService method registerApplicationMaster.

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(RegisterApplicationMasterRequest request) throws YarnException, IOException {
    AMRMTokenIdentifier amrmTokenIdentifier = YarnServerSecurityUtils.authorizeRequest();
    ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.getApplicationAttemptId();
    ApplicationId appID = applicationAttemptId.getApplicationId();
    AllocateResponseLock lock = responseMap.get(applicationAttemptId);
    if (lock == null) {
        RMAuditLogger.logFailure(this.rmContext.getRMApps().get(appID).getUser(), AuditConstants.REGISTER_AM, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService", "Error in registering application master", appID, applicationAttemptId);
        throwApplicationDoesNotExistInCacheException(applicationAttemptId);
    }
    // Allow only one thread in AM to do registerApp at a time.
    synchronized (lock) {
        AllocateResponse lastResponse = lock.getAllocateResponse();
        if (hasApplicationMasterRegistered(applicationAttemptId)) {
            String message = "Application Master is already registered : " + appID;
            LOG.warn(message);
            RMAuditLogger.logFailure(this.rmContext.getRMApps().get(appID).getUser(), AuditConstants.REGISTER_AM, "", "ApplicationMasterService", message, appID, applicationAttemptId);
            throw new InvalidApplicationMasterRequestException(message);
        }
        this.amLivelinessMonitor.receivedPing(applicationAttemptId);
        RMApp app = this.rmContext.getRMApps().get(appID);
        // Setting the response id to 0 to identify if the
        // application master is register for the respective attemptid
        lastResponse.setResponseId(0);
        lock.setAllocateResponse(lastResponse);
        LOG.info("AM registration " + applicationAttemptId);
        this.rmContext.getDispatcher().getEventHandler().handle(new RMAppAttemptRegistrationEvent(applicationAttemptId, request.getHost(), request.getRpcPort(), request.getTrackingUrl()));
        RMAuditLogger.logSuccess(app.getUser(), AuditConstants.REGISTER_AM, "ApplicationMasterService", appID, applicationAttemptId);
        // Pick up min/max resource from scheduler...
        RegisterApplicationMasterResponse response = recordFactory.newRecordInstance(RegisterApplicationMasterResponse.class);
        response.setMaximumResourceCapability(rScheduler.getMaximumResourceCapability(app.getQueue()));
        response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId).getSubmissionContext().getAMContainerSpec().getApplicationACLs());
        response.setQueue(app.getQueue());
        if (UserGroupInformation.isSecurityEnabled()) {
            LOG.info("Setting client token master key");
            response.setClientToAMTokenMasterKey(java.nio.ByteBuffer.wrap(rmContext.getClientToAMTokenSecretManager().getMasterKey(applicationAttemptId).getEncoded()));
        }
        // and corresponding NM tokens.
        if (app.getApplicationSubmissionContext().getKeepContainersAcrossApplicationAttempts()) {
            List<Container> transferredContainers = rScheduler.getTransferredContainers(applicationAttemptId);
            if (!transferredContainers.isEmpty()) {
                response.setContainersFromPreviousAttempts(transferredContainers);
                List<NMToken> nmTokens = new ArrayList<NMToken>();
                for (Container container : transferredContainers) {
                    try {
                        NMToken token = rmContext.getNMTokenSecretManager().createAndGetNMToken(app.getUser(), applicationAttemptId, container);
                        if (null != token) {
                            nmTokens.add(token);
                        }
                    } catch (IllegalArgumentException e) {
                        // will be automatically retried by RMProxy in RPC layer.
                        if (e.getCause() instanceof UnknownHostException) {
                            throw (UnknownHostException) e.getCause();
                        }
                    }
                }
                response.setNMTokensFromPreviousAttempts(nmTokens);
                LOG.info("Application " + appID + " retrieved " + transferredContainers.size() + " containers from previous" + " attempts and " + nmTokens.size() + " NM tokens.");
            }
        }
        response.setSchedulerResourceTypes(rScheduler.getSchedulingResourceTypes());
        return response;
    }
}

Example 33 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class MockAM method schedule.

public AllocateResponse schedule() throws Exception {
    AllocateResponse response = allocate(requests, releases);
    requests.clear();
    releases.clear();
    return response;
}

Example 34 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class MRAMSimulator method sendContainerRequest.

@Override
protected void sendContainerRequest() throws YarnException, IOException, InterruptedException {
    if (isFinished) {
        return;
    }
    // send out request
    List<ResourceRequest> ask = null;
    if (mapFinished != mapTotal) {
        // map phase
        if (!pendingMaps.isEmpty()) {
            ask = packageRequests(mergeLists(pendingMaps, scheduledMaps), PRIORITY_MAP);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "request for {1} mappers.", appId, pendingMaps.size()));
            scheduledMaps.addAll(pendingMaps);
            pendingMaps.clear();
        } else if (!pendingFailedMaps.isEmpty()) {
            ask = packageRequests(mergeLists(pendingFailedMaps, scheduledMaps), PRIORITY_MAP);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "requests for {1} failed mappers.", appId, pendingFailedMaps.size()));
            scheduledMaps.addAll(pendingFailedMaps);
            pendingFailedMaps.clear();
        }
    } else if (reduceFinished != reduceTotal) {
        // reduce phase
        if (!pendingReduces.isEmpty()) {
            ask = packageRequests(mergeLists(pendingReduces, scheduledReduces), PRIORITY_REDUCE);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "requests for {1} reducers.", appId, pendingReduces.size()));
            scheduledReduces.addAll(pendingReduces);
            pendingReduces.clear();
        } else if (!pendingFailedReduces.isEmpty()) {
            ask = packageRequests(mergeLists(pendingFailedReduces, scheduledReduces), PRIORITY_REDUCE);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "request for {1} failed reducers.", appId, pendingFailedReduces.size()));
            scheduledReduces.addAll(pendingFailedReduces);
            pendingFailedReduces.clear();
        }
    }
    if (ask == null) {
        ask = new ArrayList<>();
    }
    final AllocateRequest request = createAllocateRequest(ask);
    if (totalContainers == 0) {
        request.setProgress(1.0f);
    } else {
        request.setProgress((float) finishedContainers / totalContainers);
    }
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(appAttemptId.toString());
    Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    AllocateResponse response = ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() {

        @Override
        public AllocateResponse run() throws Exception {
            return rm.getApplicationMasterService().allocate(request);
        }
    });
    if (response != null) {
        responseQueue.put(response);
    }
}

Example 35 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class MRAMSimulator method processResponseQueue.

@Override
@SuppressWarnings("unchecked")
protected void processResponseQueue() throws Exception {
    while (!responseQueue.isEmpty()) {
        AllocateResponse response = responseQueue.take();
        // check completed containers
        if (!response.getCompletedContainersStatuses().isEmpty()) {
            for (ContainerStatus cs : response.getCompletedContainersStatuses()) {
                ContainerId containerId = cs.getContainerId();
                if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) {
                    if (assignedMaps.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one" + "mapper finished ({1}).", appId, containerId));
                        assignedMaps.remove(containerId);
                        mapFinished++;
                        finishedContainers++;
                    } else if (assignedReduces.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one" + "reducer finished ({1}).", appId, containerId));
                        assignedReduces.remove(containerId);
                        reduceFinished++;
                        finishedContainers++;
                    } else if (amContainer.getId().equals(containerId)) {
                        // am container released event
                        isFinished = true;
                        LOG.info(MessageFormat.format("Application {0} goes to " + "finish.", appId));
                    }
                    if (mapFinished >= mapTotal && reduceFinished >= reduceTotal) {
                        lastStep();
                    }
                } else {
                    // container to be killed
                    if (assignedMaps.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one " + "mapper killed ({1}).", appId, containerId));
                        pendingFailedMaps.add(assignedMaps.remove(containerId));
                    } else if (assignedReduces.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one " + "reducer killed ({1}).", appId, containerId));
                        pendingFailedReduces.add(assignedReduces.remove(containerId));
                    } else if (amContainer.getId().equals(containerId)) {
                        LOG.info(MessageFormat.format("Application {0}'s AM is " + "going to be killed. Waiting for rescheduling...", appId));
                    }
                }
            }
        }
        // check finished
        if (isAMContainerRunning && (mapFinished >= mapTotal) && (reduceFinished >= reduceTotal)) {
            isAMContainerRunning = false;
            LOG.debug(MessageFormat.format("Application {0} sends out event " + "to clean up its AM container.", appId));
            isFinished = true;
            break;
        }
        // check allocated containers
        for (Container container : response.getAllocatedContainers()) {
            if (!scheduledMaps.isEmpty()) {
                ContainerSimulator cs = scheduledMaps.remove();
                LOG.debug(MessageFormat.format("Application {0} starts a " + "launch a mapper ({1}).", appId, container.getId()));
                assignedMaps.put(container.getId(), cs);
                se.getNmMap().get(container.getNodeId()).addNewContainer(container, cs.getLifeTime());
            } else if (!this.scheduledReduces.isEmpty()) {
                ContainerSimulator cs = scheduledReduces.remove();
                LOG.debug(MessageFormat.format("Application {0} starts a " + "launch a reducer ({1}).", appId, container.getId()));
                assignedReduces.put(container.getId(), cs);
                se.getNmMap().get(container.getNodeId()).addNewContainer(container, cs.getLifeTime());
            }
        }
    }
}