Examples with NMToken org.apache.hadoop.yarn.api.records.NMToken used on opensource projects

Example 6 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class TestDistributedScheduling method testAMRMClient.

/**
   * Validates if AMRMClient can be used with Distributed Scheduling turned on.
   *
   * @throws Exception
   */
@Test(timeout = 120000)
@SuppressWarnings("unchecked")
public void testAMRMClient() throws Exception {
    AMRMClientImpl<AMRMClient.ContainerRequest> amClient = null;
    try {
        Priority priority = Priority.newInstance(1);
        Priority priority2 = Priority.newInstance(2);
        Resource capability = Resource.newInstance(1024, 1);
        List<NodeReport> nodeReports = rmClient.getNodeReports(NodeState.RUNNING);
        String node = nodeReports.get(0).getNodeId().getHost();
        String rack = nodeReports.get(0).getRackName();
        String[] nodes = new String[] { node };
        String[] racks = new String[] { rack };
        // start am rm client
        amClient = new AMRMClientImpl(client);
        amClient.init(yarnConf);
        amClient.start();
        amClient.registerApplicationMaster(NetUtils.getHostname(), 1024, "");
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        RemoteRequestsTable<ContainerRequest> remoteRequestsTable = amClient.getTable(0);
        int containersRequestedNode = remoteRequestsTable.get(priority, node, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int containersRequestedRack = remoteRequestsTable.get(priority, rack, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int containersRequestedAny = remoteRequestsTable.get(priority, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int oppContainersRequestedAny = remoteRequestsTable.get(priority2, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC, capability).remoteRequest.getNumContainers();
        assertEquals(2, containersRequestedNode);
        assertEquals(2, containersRequestedRack);
        assertEquals(2, containersRequestedAny);
        assertEquals(1, oppContainersRequestedAny);
        assertEquals(4, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        // RM should allocate container within 2 calls to allocate()
        int allocatedContainerCount = 0;
        int iterationsLeft = 10;
        Set<ContainerId> releases = new TreeSet<>();
        amClient.getNMTokenCache().clearCache();
        Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
        HashMap<String, Token> receivedNMTokens = new HashMap<>();
        while (allocatedContainerCount < (containersRequestedAny + oppContainersRequestedAny) && iterationsLeft-- > 0) {
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            assertEquals(0, amClient.ask.size());
            assertEquals(0, amClient.release.size());
            allocatedContainerCount += allocResponse.getAllocatedContainers().size();
            for (Container container : allocResponse.getAllocatedContainers()) {
                ContainerId rejectContainerId = container.getId();
                releases.add(rejectContainerId);
            }
            for (NMToken token : allocResponse.getNMTokens()) {
                String nodeID = token.getNodeId().toString();
                receivedNMTokens.put(nodeID, token.getToken());
            }
            if (allocatedContainerCount < containersRequestedAny) {
                // sleep to let NM's heartbeat to RM and trigger allocations
                sleep(100);
            }
        }
        assertEquals(allocatedContainerCount, containersRequestedAny + oppContainersRequestedAny);
        for (ContainerId rejectContainerId : releases) {
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        assertEquals(3, amClient.release.size());
        assertEquals(0, amClient.ask.size());
        // need to tell the AMRMClient that we dont need these resources anymore
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        assertEquals(4, amClient.ask.size());
        // test RPC exception handling
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        final AMRMClient amc = amClient;
        ApplicationMasterProtocol realRM = amClient.rmClient;
        try {
            ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
            when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(new Answer<AllocateResponse>() {

                public AllocateResponse answer(InvocationOnMock invocation) throws Exception {
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
                    throw new Exception();
                }
            });
            amClient.rmClient = mockRM;
            amClient.allocate(0.1f);
        } catch (Exception ioe) {
        } finally {
            amClient.rmClient = realRM;
        }
        assertEquals(3, amClient.release.size());
        assertEquals(6, amClient.ask.size());
        iterationsLeft = 3;
        // do a few iterations to ensure RM is not going send new containers
        while (iterationsLeft-- > 0) {
            // inform RM of rejection
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            // RM did not send new containers because AM does not need any
            assertEquals(0, allocResponse.getAllocatedContainers().size());
            if (allocResponse.getCompletedContainersStatuses().size() > 0) {
                for (ContainerStatus cStatus : allocResponse.getCompletedContainersStatuses()) {
                    if (releases.contains(cStatus.getContainerId())) {
                        assertEquals(cStatus.getState(), ContainerState.COMPLETE);
                        assertEquals(-100, cStatus.getExitStatus());
                        releases.remove(cStatus.getContainerId());
                    }
                }
            }
            if (iterationsLeft > 0) {
                // sleep to make sure NM's heartbeat
                sleep(100);
            }
        }
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == Service.STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 7 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class ApplicationMasterService method registerApplicationMaster.

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(RegisterApplicationMasterRequest request) throws YarnException, IOException {
    AMRMTokenIdentifier amrmTokenIdentifier = YarnServerSecurityUtils.authorizeRequest();
    ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.getApplicationAttemptId();
    ApplicationId appID = applicationAttemptId.getApplicationId();
    AllocateResponseLock lock = responseMap.get(applicationAttemptId);
    if (lock == null) {
        RMAuditLogger.logFailure(this.rmContext.getRMApps().get(appID).getUser(), AuditConstants.REGISTER_AM, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService", "Error in registering application master", appID, applicationAttemptId);
        throwApplicationDoesNotExistInCacheException(applicationAttemptId);
    }
    // Allow only one thread in AM to do registerApp at a time.
    synchronized (lock) {
        AllocateResponse lastResponse = lock.getAllocateResponse();
        if (hasApplicationMasterRegistered(applicationAttemptId)) {
            String message = "Application Master is already registered : " + appID;
            LOG.warn(message);
            RMAuditLogger.logFailure(this.rmContext.getRMApps().get(appID).getUser(), AuditConstants.REGISTER_AM, "", "ApplicationMasterService", message, appID, applicationAttemptId);
            throw new InvalidApplicationMasterRequestException(message);
        }
        this.amLivelinessMonitor.receivedPing(applicationAttemptId);
        RMApp app = this.rmContext.getRMApps().get(appID);
        // Setting the response id to 0 to identify if the
        // application master is register for the respective attemptid
        lastResponse.setResponseId(0);
        lock.setAllocateResponse(lastResponse);
        LOG.info("AM registration " + applicationAttemptId);
        this.rmContext.getDispatcher().getEventHandler().handle(new RMAppAttemptRegistrationEvent(applicationAttemptId, request.getHost(), request.getRpcPort(), request.getTrackingUrl()));
        RMAuditLogger.logSuccess(app.getUser(), AuditConstants.REGISTER_AM, "ApplicationMasterService", appID, applicationAttemptId);
        // Pick up min/max resource from scheduler...
        RegisterApplicationMasterResponse response = recordFactory.newRecordInstance(RegisterApplicationMasterResponse.class);
        response.setMaximumResourceCapability(rScheduler.getMaximumResourceCapability(app.getQueue()));
        response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId).getSubmissionContext().getAMContainerSpec().getApplicationACLs());
        response.setQueue(app.getQueue());
        if (UserGroupInformation.isSecurityEnabled()) {
            LOG.info("Setting client token master key");
            response.setClientToAMTokenMasterKey(java.nio.ByteBuffer.wrap(rmContext.getClientToAMTokenSecretManager().getMasterKey(applicationAttemptId).getEncoded()));
        }
        // and corresponding NM tokens.
        if (app.getApplicationSubmissionContext().getKeepContainersAcrossApplicationAttempts()) {
            List<Container> transferredContainers = rScheduler.getTransferredContainers(applicationAttemptId);
            if (!transferredContainers.isEmpty()) {
                response.setContainersFromPreviousAttempts(transferredContainers);
                List<NMToken> nmTokens = new ArrayList<NMToken>();
                for (Container container : transferredContainers) {
                    try {
                        NMToken token = rmContext.getNMTokenSecretManager().createAndGetNMToken(app.getUser(), applicationAttemptId, container);
                        if (null != token) {
                            nmTokens.add(token);
                        }
                    } catch (IllegalArgumentException e) {
                        // will be automatically retried by RMProxy in RPC layer.
                        if (e.getCause() instanceof UnknownHostException) {
                            throw (UnknownHostException) e.getCause();
                        }
                    }
                }
                response.setNMTokensFromPreviousAttempts(nmTokens);
                LOG.info("Application " + appID + " retrieved " + transferredContainers.size() + " containers from previous" + " attempts and " + nmTokens.size() + " NM tokens.");
            }
        }
        response.setSchedulerResourceTypes(rScheduler.getSchedulingResourceTypes());
        return response;
    }
}

Example 8 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class RegisterApplicationMasterResponsePBImpl method initLocalNewNMTokenList.

private synchronized void initLocalNewNMTokenList() {
    RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
    List<NMTokenProto> list = p.getNmTokensFromPreviousAttemptsList();
    nmTokens = new ArrayList<NMToken>();
    for (NMTokenProto t : list) {
        nmTokens.add(convertFromProtoFormat(t));
    }
}

Example 9 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class NMTokenSecretManagerInRM method createAndGetNMToken.

public NMToken createAndGetNMToken(String applicationSubmitter, ApplicationAttemptId appAttemptId, Container container) {
    try {
        this.writeLock.lock();
        HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId);
        NMToken nmToken = null;
        if (nodeSet != null) {
            if (!nodeSet.contains(container.getNodeId())) {
                LOG.info("Sending NMToken for nodeId : " + container.getNodeId() + " for container : " + container.getId());
                Token token = createNMToken(container.getId().getApplicationAttemptId(), container.getNodeId(), applicationSubmitter);
                nmToken = NMToken.newInstance(container.getNodeId(), token);
                nodeSet.add(container.getNodeId());
            }
        }
        return nmToken;
    } finally {
        this.writeLock.unlock();
    }
}

Example 10 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class MockResourceManagerFacade method allocate.

@SuppressWarnings("deprecation")
@Override
public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException {
    if (request.getAskList() != null && request.getAskList().size() > 0 && request.getReleaseList() != null && request.getReleaseList().size() > 0) {
        Assert.fail("The mock RM implementation does not support receiving " + "askList and releaseList in the same heartbeat");
    }
    String amrmToken = getAppIdentifier();
    ArrayList<Container> containerList = new ArrayList<Container>();
    if (request.getAskList() != null) {
        for (ResourceRequest rr : request.getAskList()) {
            for (int i = 0; i < rr.getNumContainers(); i++) {
                ContainerId containerId = ContainerId.newInstance(getApplicationAttemptId(1), containerIndex.incrementAndGet());
                Container container = Records.newRecord(Container.class);
                container.setId(containerId);
                container.setPriority(rr.getPriority());
                // We don't use the node for running containers in the test cases. So
                // it is OK to hard code it to some dummy value
                NodeId nodeId = NodeId.newInstance(!Strings.isNullOrEmpty(rr.getResourceName()) ? rr.getResourceName() : "dummy", 1000);
                container.setNodeId(nodeId);
                container.setResource(rr.getCapability());
                containerList.add(container);
                synchronized (applicationContainerIdMap) {
                    // Keep track of the containers returned to this application. We
                    // will need it in future
                    Assert.assertTrue("The application id is Not registered before allocate(): " + amrmToken, applicationContainerIdMap.containsKey(amrmToken));
                    List<ContainerId> ids = applicationContainerIdMap.get(amrmToken);
                    ids.add(containerId);
                    this.allocatedContainerMap.put(containerId, container);
                }
            }
        }
    }
    if (request.getReleaseList() != null && request.getReleaseList().size() > 0) {
        Log.getLog().info("Releasing containers: " + request.getReleaseList().size());
        synchronized (applicationContainerIdMap) {
            Assert.assertTrue("The application id is not registered before allocate(): " + amrmToken, applicationContainerIdMap.containsKey(amrmToken));
            List<ContainerId> ids = applicationContainerIdMap.get(amrmToken);
            for (ContainerId id : request.getReleaseList()) {
                boolean found = false;
                for (ContainerId c : ids) {
                    if (c.equals(id)) {
                        found = true;
                        break;
                    }
                }
                Assert.assertTrue("ContainerId " + id + " being released is not valid for application: " + conf.get("AMRMTOKEN"), found);
                ids.remove(id);
                // Return the released container back to the AM with new fake Ids. The
                // test case does not care about the IDs. The IDs are faked because
                // otherwise the LRM will throw duplication identifier exception. This
                // returning of fake containers is ONLY done for testing purpose - for
                // the test code to get confirmation that the sub-cluster resource
                // managers received the release request
                ContainerId fakeContainerId = ContainerId.newInstance(getApplicationAttemptId(1), containerIndex.incrementAndGet());
                Container fakeContainer = allocatedContainerMap.get(id);
                fakeContainer.setId(fakeContainerId);
                containerList.add(fakeContainer);
            }
        }
    }
    Log.getLog().info("Allocating containers: " + containerList.size() + " for application attempt: " + conf.get("AMRMTOKEN"));
    // Always issue a new AMRMToken as if RM rolled master key
    Token newAMRMToken = Token.newInstance(new byte[0], "", new byte[0], "");
    return AllocateResponse.newInstance(0, new ArrayList<ContainerStatus>(), containerList, new ArrayList<NodeReport>(), null, AMCommand.AM_RESYNC, 1, null, new ArrayList<NMToken>(), newAMRMToken, new ArrayList<UpdatedContainer>());
}