Examples with AllocateRequest org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest used on opensource projects

Example 1 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class RMContainerRequestor method makeRemoteRequest.

protected AllocateResponse makeRemoteRequest() throws YarnException, IOException {
    applyRequestLimits();
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions), new ArrayList<String>(blacklistRemovals));
    AllocateRequest allocateRequest = AllocateRequest.newInstance(lastResponseID, super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>(release), blacklistRequest);
    AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
    lastResponseID = allocateResponse.getResponseId();
    availableResources = allocateResponse.getAvailableResources();
    lastClusterNmCount = clusterNmCount;
    clusterNmCount = allocateResponse.getNumClusterNodes();
    int numCompletedContainers = allocateResponse.getCompletedContainersStatuses().size();
    if (ask.size() > 0 || release.size() > 0) {
        LOG.info("getResources() for " + applicationId + ":" + " ask=" + ask.size() + " release= " + release.size() + " newContainers=" + allocateResponse.getAllocatedContainers().size() + " finishedContainers=" + numCompletedContainers + " resourcelimit=" + availableResources + " knownNMs=" + clusterNmCount);
    }
    ask.clear();
    release.clear();
    if (numCompletedContainers > 0) {
        // re-send limited requests when a container completes to trigger asking
        // for more containers
        requestLimitsToUpdate.addAll(requestLimits.keySet());
    }
    if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
        LOG.info("Update the blacklist for " + applicationId + ": blacklistAdditions=" + blacklistAdditions.size() + " blacklistRemovals=" + blacklistRemovals.size());
    }
    blacklistAdditions.clear();
    blacklistRemovals.clear();
    return allocateResponse;
}

Example 2 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMClient method testAMRMClientOnAMRMTokenRollOver.

@Test(timeout = 60000)
public void testAMRMClientOnAMRMTokenRollOver() throws YarnException, IOException {
    AMRMClient<ContainerRequest> amClient = null;
    try {
        AMRMTokenSecretManager amrmTokenSecretManager = yarnCluster.getResourceManager().getRMContext().getAMRMTokenSecretManager();
        // start am rm client
        amClient = AMRMClient.<ContainerRequest>createAMRMClient();
        amClient.init(conf);
        amClient.start();
        Long startTime = System.currentTimeMillis();
        amClient.registerApplicationMaster("Host", 10000, "");
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1 = getAMRMToken();
        Assert.assertNotNull(amrmToken_1);
        Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(), amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
        // At mean time, the old AMRMToken should continue to work
        while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
            amClient.allocate(0.1f);
            sleep(1000);
        }
        amClient.allocate(0.1f);
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2 = getAMRMToken();
        Assert.assertNotNull(amrmToken_2);
        Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(), amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
        Assert.assertNotEquals(amrmToken_1, amrmToken_2);
        // can do the allocate call with latest AMRMToken
        AllocateResponse response = amClient.allocate(0.1f);
        // Verify latest AMRMToken can be used to send allocation request.
        UserGroupInformation testUser1 = UserGroupInformation.createRemoteUser("testUser1");
        AMRMTokenIdentifierForTest newVersionTokenIdentifier = new AMRMTokenIdentifierForTest(amrmToken_2.decodeIdentifier(), "message");
        Assert.assertEquals("Message is changed after set to newVersionTokenIdentifier", "message", newVersionTokenIdentifier.getMessage());
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newVersionToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(newVersionTokenIdentifier.getBytes(), amrmTokenSecretManager.retrievePassword(newVersionTokenIdentifier), newVersionTokenIdentifier.getKind(), new Text());
        SecurityUtil.setTokenService(newVersionToken, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress());
        testUser1.addToken(newVersionToken);
        AllocateRequest request = Records.newRecord(AllocateRequest.class);
        request.setResponseId(response.getResponseId());
        testUser1.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

            @Override
            public ApplicationMasterProtocol run() {
                return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress(), conf);
            }
        }).allocate(request);
        // and can not use this rolled-over token to make a allocate all.
        while (true) {
            if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
                if (amrmTokenSecretManager.getNextMasterKeyData() == null) {
                    break;
                } else if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getNextMasterKeyData().getMasterKey().getKeyId()) {
                    break;
                }
            }
            amClient.allocate(0.1f);
            sleep(1000);
        }
        try {
            UserGroupInformation testUser2 = UserGroupInformation.createRemoteUser("testUser2");
            SecurityUtil.setTokenService(amrmToken_2, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress());
            testUser2.addToken(amrmToken_2);
            testUser2.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

                @Override
                public ApplicationMasterProtocol run() {
                    return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress(), conf);
                }
            }).allocate(Records.newRecord(AllocateRequest.class));
            Assert.fail("The old Token should not work");
        } catch (Exception ex) {
            Assert.assertTrue(ex instanceof InvalidToken);
            Assert.assertTrue(ex.getMessage().contains("Invalid AMRMToken from " + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
        }
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 3 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class AMSimulator method createAllocateRequest.

protected AllocateRequest createAllocateRequest(List<ResourceRequest> ask, List<ContainerId> toRelease) {
    AllocateRequest allocateRequest = recordFactory.newRecordInstance(AllocateRequest.class);
    allocateRequest.setResponseId(RESPONSE_ID++);
    allocateRequest.setAskList(ask);
    allocateRequest.setReleaseList(toRelease);
    return allocateRequest;
}

Example 4 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class MRAMSimulator method sendContainerRequest.

@Override
protected void sendContainerRequest() throws YarnException, IOException, InterruptedException {
    if (isFinished) {
        return;
    }
    // send out request
    List<ResourceRequest> ask = null;
    if (mapFinished != mapTotal) {
        // map phase
        if (!pendingMaps.isEmpty()) {
            ask = packageRequests(mergeLists(pendingMaps, scheduledMaps), PRIORITY_MAP);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "request for {1} mappers.", appId, pendingMaps.size()));
            scheduledMaps.addAll(pendingMaps);
            pendingMaps.clear();
        } else if (!pendingFailedMaps.isEmpty()) {
            ask = packageRequests(mergeLists(pendingFailedMaps, scheduledMaps), PRIORITY_MAP);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "requests for {1} failed mappers.", appId, pendingFailedMaps.size()));
            scheduledMaps.addAll(pendingFailedMaps);
            pendingFailedMaps.clear();
        }
    } else if (reduceFinished != reduceTotal) {
        // reduce phase
        if (!pendingReduces.isEmpty()) {
            ask = packageRequests(mergeLists(pendingReduces, scheduledReduces), PRIORITY_REDUCE);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "requests for {1} reducers.", appId, pendingReduces.size()));
            scheduledReduces.addAll(pendingReduces);
            pendingReduces.clear();
        } else if (!pendingFailedReduces.isEmpty()) {
            ask = packageRequests(mergeLists(pendingFailedReduces, scheduledReduces), PRIORITY_REDUCE);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "request for {1} failed reducers.", appId, pendingFailedReduces.size()));
            scheduledReduces.addAll(pendingFailedReduces);
            pendingFailedReduces.clear();
        }
    }
    if (ask == null) {
        ask = new ArrayList<>();
    }
    final AllocateRequest request = createAllocateRequest(ask);
    if (totalContainers == 0) {
        request.setProgress(1.0f);
    } else {
        request.setProgress((float) finishedContainers / totalContainers);
    }
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(appAttemptId.toString());
    Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    AllocateResponse response = ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() {

        @Override
        public AllocateResponse run() throws Exception {
            return rm.getApplicationMasterService().allocate(request);
        }
    });
    if (response != null) {
        responseQueue.put(response);
    }
}

Example 5 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestOpportunisticContainerAllocatorAMService method createService.

private OpportunisticContainerAllocatorAMService createService(final RecordFactory factory, final RMContext rmContext, final Container c) {
    return new OpportunisticContainerAllocatorAMService(rmContext, null) {

        @Override
        public RegisterApplicationMasterResponse registerApplicationMaster(RegisterApplicationMasterRequest request) throws YarnException, IOException {
            RegisterApplicationMasterResponse resp = factory.newRecordInstance(RegisterApplicationMasterResponse.class);
            // Dummy Entry to Assert that we get this object back
            resp.setQueue("dummyQueue");
            return resp;
        }

        @Override
        public FinishApplicationMasterResponse finishApplicationMaster(FinishApplicationMasterRequest request) throws YarnException, IOException {
            FinishApplicationMasterResponse resp = factory.newRecordInstance(FinishApplicationMasterResponse.class);
            // Dummy Entry to Assert that we get this object back
            resp.setIsUnregistered(false);
            return resp;
        }

        @Override
        public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException {
            AllocateResponse response = factory.newRecordInstance(AllocateResponse.class);
            response.setNumClusterNodes(12345);
            response.setAllocatedContainers(Arrays.asList(c));
            return response;
        }

        @Override
        public RegisterDistributedSchedulingAMResponse registerApplicationMasterForDistributedScheduling(RegisterApplicationMasterRequest request) throws YarnException, IOException {
            RegisterDistributedSchedulingAMResponse resp = factory.newRecordInstance(RegisterDistributedSchedulingAMResponse.class);
            resp.setContainerIdStart(54321L);
            resp.setMaxContainerResource(Resource.newInstance(4096, 4));
            resp.setMinContainerResource(Resource.newInstance(1024, 1));
            resp.setIncrContainerResource(Resource.newInstance(2048, 2));
            return resp;
        }

        @Override
        public DistributedSchedulingAllocateResponse allocateForDistributedScheduling(DistributedSchedulingAllocateRequest request) throws YarnException, IOException {
            List<ResourceRequest> askList = request.getAllocateRequest().getAskList();
            List<Container> allocatedContainers = request.getAllocatedContainers();
            Assert.assertEquals(1, allocatedContainers.size());
            Assert.assertEquals(ExecutionType.OPPORTUNISTIC, allocatedContainers.get(0).getExecutionType());
            Assert.assertEquals(1, askList.size());
            Assert.assertTrue(askList.get(0).getExecutionTypeRequest().getEnforceExecutionType());
            DistributedSchedulingAllocateResponse resp = factory.newRecordInstance(DistributedSchedulingAllocateResponse.class);
            resp.setNodesForScheduling(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h1", 1234), "http://h1:4321")));
            return resp;
        }
    };
}