Examples with AllocateResponse org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse used on opensource projects

Example 21 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClient method allocateAndStartContainers.

private List<Container> allocateAndStartContainers(final AMRMClient<ContainerRequest> amClient, final NMClient nmClient, int num) throws YarnException, IOException {
    // set up allocation requests
    for (int i = 0; i < num; ++i) {
        amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    }
    // send allocation requests
    amClient.allocate(0.1f);
    // let NM heartbeat to RM and trigger allocations
    triggerSchedulingWithNMHeartBeat();
    // get allocations
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    List<Container> containers = allocResponse.getAllocatedContainers();
    Assert.assertEquals(num, containers.size());
    // build container launch context
    Credentials ts = new Credentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    ts.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    // start a process long enough for increase/decrease action to take effect
    ContainerLaunchContext clc = BuilderUtils.newContainerLaunchContext(Collections.<String, LocalResource>emptyMap(), new HashMap<String, String>(), Arrays.asList("sleep", "100"), new HashMap<String, ByteBuffer>(), securityTokens, new HashMap<ApplicationAccessType, String>());
    // start the containers and make sure they are in RUNNING state
    try {
        for (int i = 0; i < num; i++) {
            Container container = containers.get(i);
            nmClient.startContainer(container, clc);
            // container status
            while (true) {
                ContainerStatus status = nmClient.getContainerStatus(container.getId(), container.getNodeId());
                if (status.getState() == ContainerState.RUNNING) {
                    break;
                }
                sleep(10);
            }
        }
    } catch (YarnException e) {
        throw new AssertionError("Exception is not expected: " + e);
    }
    // let NM's heartbeat to RM to confirm container launch
    triggerSchedulingWithNMHeartBeat();
    return containers;
}

Example 22 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClient method testAMRMClientOnAMRMTokenRollOver.

@Test(timeout = 60000)
public void testAMRMClientOnAMRMTokenRollOver() throws YarnException, IOException {
    AMRMClient<ContainerRequest> amClient = null;
    try {
        AMRMTokenSecretManager amrmTokenSecretManager = yarnCluster.getResourceManager().getRMContext().getAMRMTokenSecretManager();
        // start am rm client
        amClient = AMRMClient.<ContainerRequest>createAMRMClient();
        amClient.init(conf);
        amClient.start();
        Long startTime = System.currentTimeMillis();
        amClient.registerApplicationMaster("Host", 10000, "");
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1 = getAMRMToken();
        Assert.assertNotNull(amrmToken_1);
        Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(), amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
        // At mean time, the old AMRMToken should continue to work
        while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
            amClient.allocate(0.1f);
            sleep(1000);
        }
        amClient.allocate(0.1f);
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2 = getAMRMToken();
        Assert.assertNotNull(amrmToken_2);
        Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(), amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
        Assert.assertNotEquals(amrmToken_1, amrmToken_2);
        // can do the allocate call with latest AMRMToken
        AllocateResponse response = amClient.allocate(0.1f);
        // Verify latest AMRMToken can be used to send allocation request.
        UserGroupInformation testUser1 = UserGroupInformation.createRemoteUser("testUser1");
        AMRMTokenIdentifierForTest newVersionTokenIdentifier = new AMRMTokenIdentifierForTest(amrmToken_2.decodeIdentifier(), "message");
        Assert.assertEquals("Message is changed after set to newVersionTokenIdentifier", "message", newVersionTokenIdentifier.getMessage());
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newVersionToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(newVersionTokenIdentifier.getBytes(), amrmTokenSecretManager.retrievePassword(newVersionTokenIdentifier), newVersionTokenIdentifier.getKind(), new Text());
        SecurityUtil.setTokenService(newVersionToken, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress());
        testUser1.addToken(newVersionToken);
        AllocateRequest request = Records.newRecord(AllocateRequest.class);
        request.setResponseId(response.getResponseId());
        testUser1.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

            @Override
            public ApplicationMasterProtocol run() {
                return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress(), conf);
            }
        }).allocate(request);
        // and can not use this rolled-over token to make a allocate all.
        while (true) {
            if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
                if (amrmTokenSecretManager.getNextMasterKeyData() == null) {
                    break;
                } else if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getNextMasterKeyData().getMasterKey().getKeyId()) {
                    break;
                }
            }
            amClient.allocate(0.1f);
            sleep(1000);
        }
        try {
            UserGroupInformation testUser2 = UserGroupInformation.createRemoteUser("testUser2");
            SecurityUtil.setTokenService(amrmToken_2, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress());
            testUser2.addToken(amrmToken_2);
            testUser2.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

                @Override
                public ApplicationMasterProtocol run() {
                    return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress(), conf);
                }
            }).allocate(Records.newRecord(AllocateRequest.class));
            Assert.fail("The old Token should not work");
        } catch (Exception ex) {
            Assert.assertTrue(ex instanceof InvalidToken);
            Assert.assertTrue(ex.getMessage().contains("Invalid AMRMToken from " + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
        }
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 23 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClient method testAMRMClientMatchStorage.

//(timeout=60000)
@Test
public void testAMRMClientMatchStorage() throws YarnException, IOException {
    AMRMClientImpl<ContainerRequest> amClient = null;
    try {
        // start am rm client
        amClient = (AMRMClientImpl<ContainerRequest>) AMRMClient.<ContainerRequest>createAMRMClient();
        amClient.init(conf);
        amClient.start();
        amClient.registerApplicationMaster("Host", 10000, "");
        Priority priority1 = Records.newRecord(Priority.class);
        priority1.setPriority(2);
        ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, racks, priority);
        ContainerRequest storedContainer2 = new ContainerRequest(capability, nodes, racks, priority);
        ContainerRequest storedContainer3 = new ContainerRequest(capability, null, null, priority1);
        amClient.addContainerRequest(storedContainer1);
        amClient.addContainerRequest(storedContainer2);
        amClient.addContainerRequest(storedContainer3);
        // test addition and storage
        RemoteRequestsTable<ContainerRequest> remoteRequestsTable = amClient.getTable(0);
        int containersRequestedAny = remoteRequestsTable.get(priority, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        assertEquals(2, containersRequestedAny);
        containersRequestedAny = remoteRequestsTable.get(priority1, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        assertEquals(1, containersRequestedAny);
        List<? extends Collection<ContainerRequest>> matches = amClient.getMatchingRequests(priority, node, capability);
        verifyMatches(matches, 2);
        matches = amClient.getMatchingRequests(priority, rack, capability);
        verifyMatches(matches, 2);
        matches = amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability);
        verifyMatches(matches, 2);
        matches = amClient.getMatchingRequests(priority1, rack, capability);
        assertTrue(matches.isEmpty());
        matches = amClient.getMatchingRequests(priority1, ResourceRequest.ANY, capability);
        verifyMatches(matches, 1);
        // test removal
        amClient.removeContainerRequest(storedContainer3);
        matches = amClient.getMatchingRequests(priority, node, capability);
        verifyMatches(matches, 2);
        amClient.removeContainerRequest(storedContainer2);
        matches = amClient.getMatchingRequests(priority, node, capability);
        verifyMatches(matches, 1);
        matches = amClient.getMatchingRequests(priority, rack, capability);
        verifyMatches(matches, 1);
        // test matching of containers
        ContainerRequest storedRequest = matches.get(0).iterator().next();
        assertEquals(storedContainer1, storedRequest);
        amClient.removeContainerRequest(storedContainer1);
        matches = amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability);
        assertTrue(matches.isEmpty());
        matches = amClient.getMatchingRequests(priority1, ResourceRequest.ANY, capability);
        assertTrue(matches.isEmpty());
        // 0 requests left. everything got cleaned up
        assertTrue(amClient.getTable(0).isEmpty());
        // go through an exemplary allocation, matching and release cycle
        amClient.addContainerRequest(storedContainer1);
        amClient.addContainerRequest(storedContainer3);
        // RM should allocate container within 2 calls to allocate()
        int allocatedContainerCount = 0;
        int iterationsLeft = 3;
        while (allocatedContainerCount < 2 && iterationsLeft-- > 0) {
            Log.getLog().info("Allocated " + allocatedContainerCount + " containers" + " with " + iterationsLeft + " iterations left");
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            assertEquals(0, amClient.ask.size());
            assertEquals(0, amClient.release.size());
            assertEquals(nodeCount, amClient.getClusterNodeCount());
            allocatedContainerCount += allocResponse.getAllocatedContainers().size();
            for (Container container : allocResponse.getAllocatedContainers()) {
                ContainerRequest expectedRequest = container.getPriority().equals(storedContainer1.getPriority()) ? storedContainer1 : storedContainer3;
                matches = amClient.getMatchingRequests(container.getPriority(), ResourceRequest.ANY, container.getResource());
                // test correct matched container is returned
                verifyMatches(matches, 1);
                ContainerRequest matchedRequest = matches.get(0).iterator().next();
                assertEquals(matchedRequest, expectedRequest);
                amClient.removeContainerRequest(matchedRequest);
                // assign this container, use it and release it
                amClient.releaseAssignedContainer(container.getId());
            }
            if (allocatedContainerCount < containersRequestedAny) {
                // let NM heartbeat to RM and trigger allocations
                triggerSchedulingWithNMHeartBeat();
            }
        }
        assertEquals(2, allocatedContainerCount);
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.release.size());
        assertEquals(0, amClient.ask.size());
        assertEquals(0, allocResponse.getAllocatedContainers().size());
        // 0 requests left. everything got cleaned up
        assertTrue(remoteRequestsTable.isEmpty());
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 24 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClient method testAllocation.

private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient) throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    assertNumContainers(amClient, 0, 2, 2, 2, 3, 0);
    int containersRequestedAny = 2;
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    int iterationsLeft = 3;
    Set<ContainerId> releases = new TreeSet<ContainerId>();
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
    while (allocatedContainerCount < containersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        assertEquals(nodeCount, amClient.getClusterNodeCount());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        for (Container container : allocResponse.getAllocatedContainers()) {
            ContainerId rejectContainerId = container.getId();
            releases.add(rejectContainerId);
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            if (receivedNMTokens.containsKey(nodeID)) {
                Assert.fail("Received token again for : " + nodeID);
            }
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < containersRequestedAny) {
            // let NM heartbeat to RM and trigger allocations
            triggerSchedulingWithNMHeartBeat();
        }
    }
    // Should receive atleast 1 token
    Assert.assertTrue(receivedNMTokens.size() > 0 && receivedNMTokens.size() <= nodeCount);
    assertEquals(allocatedContainerCount, containersRequestedAny);
    assertEquals(2, amClient.release.size());
    assertEquals(0, amClient.ask.size());
    // need to tell the AMRMClient that we dont need these resources anymore
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    assertEquals(3, amClient.ask.size());
    // send 0 container count request for resources that are no longer needed
    ResourceRequest snoopRequest = amClient.ask.iterator().next();
    assertEquals(0, snoopRequest.getNumContainers());
    // test RPC exception handling
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    snoopRequest = amClient.ask.iterator().next();
    assertEquals(2, snoopRequest.getNumContainers());
    ApplicationMasterProtocol realRM = amClient.rmClient;
    try {
        ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
        when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(new Answer<AllocateResponse>() {

            public AllocateResponse answer(InvocationOnMock invocation) throws Exception {
                amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
                amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
                throw new Exception();
            }
        });
        amClient.rmClient = mockRM;
        amClient.allocate(0.1f);
    } catch (Exception ioe) {
    } finally {
        amClient.rmClient = realRM;
    }
    assertEquals(2, amClient.release.size());
    assertEquals(3, amClient.ask.size());
    snoopRequest = amClient.ask.iterator().next();
    // verify that the remove request made in between makeRequest and allocate 
    // has not been lost
    assertEquals(0, snoopRequest.getNumContainers());
    waitForContainerCompletion(3, amClient, releases);
}

Example 25 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClient method doContainerResourceChange.

private void doContainerResourceChange(final AMRMClient<ContainerRequest> amClient, List<Container> containers) throws YarnException, IOException {
    Assert.assertEquals(3, containers.size());
    // remember the container IDs
    Container container1 = containers.get(0);
    Container container2 = containers.get(1);
    Container container3 = containers.get(2);
    AMRMClientImpl<ContainerRequest> amClientImpl = (AMRMClientImpl<ContainerRequest>) amClient;
    Assert.assertEquals(0, amClientImpl.change.size());
    // verify newer request overwrites older request for the container1
    amClientImpl.requestContainerUpdate(container1, UpdateContainerRequest.newInstance(container1.getVersion(), container1.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(2048, 1), null));
    amClientImpl.requestContainerUpdate(container1, UpdateContainerRequest.newInstance(container1.getVersion(), container1.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(4096, 1), null));
    Assert.assertEquals(Resource.newInstance(4096, 1), amClientImpl.change.get(container1.getId()).getValue().getCapability());
    // verify new decrease request cancels old increase request for container1
    amClientImpl.requestContainerUpdate(container1, UpdateContainerRequest.newInstance(container1.getVersion(), container1.getId(), ContainerUpdateType.DECREASE_RESOURCE, Resource.newInstance(512, 1), null));
    Assert.assertEquals(Resource.newInstance(512, 1), amClientImpl.change.get(container1.getId()).getValue().getCapability());
    // request resource increase for container2
    amClientImpl.requestContainerUpdate(container2, UpdateContainerRequest.newInstance(container2.getVersion(), container2.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(2048, 1), null));
    Assert.assertEquals(Resource.newInstance(2048, 1), amClientImpl.change.get(container2.getId()).getValue().getCapability());
    // verify release request will cancel pending change requests for the same
    // container
    amClientImpl.requestContainerUpdate(container3, UpdateContainerRequest.newInstance(container3.getVersion(), container3.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(2048, 1), null));
    Assert.assertEquals(3, amClientImpl.pendingChange.size());
    amClientImpl.releaseAssignedContainer(container3.getId());
    Assert.assertEquals(2, amClientImpl.pendingChange.size());
    // as of now: container1 asks to decrease to (512, 1)
    //            container2 asks to increase to (2048, 1)
    // send allocation requests
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    Assert.assertEquals(0, amClientImpl.change.size());
    // we should get decrease confirmation right away
    List<UpdatedContainer> updatedContainers = allocResponse.getUpdatedContainers();
    Assert.assertEquals(1, updatedContainers.size());
    // we should get increase allocation after the next NM's heartbeat to RM
    triggerSchedulingWithNMHeartBeat();
    // get allocations
    allocResponse = amClient.allocate(0.1f);
    updatedContainers = allocResponse.getUpdatedContainers();
    Assert.assertEquals(1, updatedContainers.size());
}