Examples with ContainerRequest org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest used on opensource projects

Example 1 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAskWithInvalidNodeLabels.

@Test(timeout = 30000)
public void testAskWithInvalidNodeLabels() {
    AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
    // specified exp with more than one node labels
    verifyAddRequestFailed(client, new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x && y"));
}

Example 2 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method allocateAndStartContainers.

private List<Container> allocateAndStartContainers(final AMRMClient<ContainerRequest> amClient, final NMClient nmClient, int num) throws YarnException, IOException {
    // set up allocation requests
    for (int i = 0; i < num; ++i) {
        amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    }
    // send allocation requests
    amClient.allocate(0.1f);
    // let NM heartbeat to RM and trigger allocations
    triggerSchedulingWithNMHeartBeat();
    // get allocations
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    List<Container> containers = allocResponse.getAllocatedContainers();
    Assert.assertEquals(num, containers.size());
    // build container launch context
    Credentials ts = new Credentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    ts.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    // start a process long enough for increase/decrease action to take effect
    ContainerLaunchContext clc = BuilderUtils.newContainerLaunchContext(Collections.<String, LocalResource>emptyMap(), new HashMap<String, String>(), Arrays.asList("sleep", "100"), new HashMap<String, ByteBuffer>(), securityTokens, new HashMap<ApplicationAccessType, String>());
    // start the containers and make sure they are in RUNNING state
    try {
        for (int i = 0; i < num; i++) {
            Container container = containers.get(i);
            nmClient.startContainer(container, clc);
            // container status
            while (true) {
                ContainerStatus status = nmClient.getContainerStatus(container.getId(), container.getNodeId());
                if (status.getState() == ContainerState.RUNNING) {
                    break;
                }
                sleep(10);
            }
        }
    } catch (YarnException e) {
        throw new AssertionError("Exception is not expected: " + e);
    }
    // let NM's heartbeat to RM to confirm container launch
    triggerSchedulingWithNMHeartBeat();
    return containers;
}

Example 3 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAMRMClientOnAMRMTokenRollOver.

@Test(timeout = 60000)
public void testAMRMClientOnAMRMTokenRollOver() throws YarnException, IOException {
    AMRMClient<ContainerRequest> amClient = null;
    try {
        AMRMTokenSecretManager amrmTokenSecretManager = yarnCluster.getResourceManager().getRMContext().getAMRMTokenSecretManager();
        // start am rm client
        amClient = AMRMClient.<ContainerRequest>createAMRMClient();
        amClient.init(conf);
        amClient.start();
        Long startTime = System.currentTimeMillis();
        amClient.registerApplicationMaster("Host", 10000, "");
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1 = getAMRMToken();
        Assert.assertNotNull(amrmToken_1);
        Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(), amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
        // At mean time, the old AMRMToken should continue to work
        while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
            amClient.allocate(0.1f);
            sleep(1000);
        }
        amClient.allocate(0.1f);
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2 = getAMRMToken();
        Assert.assertNotNull(amrmToken_2);
        Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(), amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
        Assert.assertNotEquals(amrmToken_1, amrmToken_2);
        // can do the allocate call with latest AMRMToken
        AllocateResponse response = amClient.allocate(0.1f);
        // Verify latest AMRMToken can be used to send allocation request.
        UserGroupInformation testUser1 = UserGroupInformation.createRemoteUser("testUser1");
        AMRMTokenIdentifierForTest newVersionTokenIdentifier = new AMRMTokenIdentifierForTest(amrmToken_2.decodeIdentifier(), "message");
        Assert.assertEquals("Message is changed after set to newVersionTokenIdentifier", "message", newVersionTokenIdentifier.getMessage());
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newVersionToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(newVersionTokenIdentifier.getBytes(), amrmTokenSecretManager.retrievePassword(newVersionTokenIdentifier), newVersionTokenIdentifier.getKind(), new Text());
        SecurityUtil.setTokenService(newVersionToken, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress());
        testUser1.addToken(newVersionToken);
        AllocateRequest request = Records.newRecord(AllocateRequest.class);
        request.setResponseId(response.getResponseId());
        testUser1.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

            @Override
            public ApplicationMasterProtocol run() {
                return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress(), conf);
            }
        }).allocate(request);
        // and can not use this rolled-over token to make a allocate all.
        while (true) {
            if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
                if (amrmTokenSecretManager.getNextMasterKeyData() == null) {
                    break;
                } else if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getNextMasterKeyData().getMasterKey().getKeyId()) {
                    break;
                }
            }
            amClient.allocate(0.1f);
            sleep(1000);
        }
        try {
            UserGroupInformation testUser2 = UserGroupInformation.createRemoteUser("testUser2");
            SecurityUtil.setTokenService(amrmToken_2, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress());
            testUser2.addToken(amrmToken_2);
            testUser2.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

                @Override
                public ApplicationMasterProtocol run() {
                    return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress(), conf);
                }
            }).allocate(Records.newRecord(AllocateRequest.class));
            Assert.fail("The old Token should not work");
        } catch (Exception ex) {
            Assert.assertTrue(ex instanceof InvalidToken);
            Assert.assertTrue(ex.getMessage().contains("Invalid AMRMToken from " + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
        }
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 4 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAMRMClientMatchingFit.

@Test(timeout = 60000)
public void testAMRMClientMatchingFit() throws YarnException, IOException {
    AMRMClient<ContainerRequest> amClient = null;
    try {
        // start am rm client
        amClient = AMRMClient.<ContainerRequest>createAMRMClient();
        amClient.init(conf);
        amClient.start();
        amClient.registerApplicationMaster("Host", 10000, "");
        Resource capability1 = Resource.newInstance(1024, 2);
        Resource capability2 = Resource.newInstance(1024, 1);
        Resource capability3 = Resource.newInstance(1000, 2);
        Resource capability4 = Resource.newInstance(2000, 1);
        Resource capability5 = Resource.newInstance(1000, 3);
        Resource capability6 = Resource.newInstance(2000, 1);
        Resource capability7 = Resource.newInstance(2000, 1);
        ContainerRequest storedContainer1 = new ContainerRequest(capability1, nodes, racks, priority);
        ContainerRequest storedContainer2 = new ContainerRequest(capability2, nodes, racks, priority);
        ContainerRequest storedContainer3 = new ContainerRequest(capability3, nodes, racks, priority);
        ContainerRequest storedContainer4 = new ContainerRequest(capability4, nodes, racks, priority);
        ContainerRequest storedContainer5 = new ContainerRequest(capability5, nodes, racks, priority);
        ContainerRequest storedContainer6 = new ContainerRequest(capability6, nodes, racks, priority);
        ContainerRequest storedContainer7 = new ContainerRequest(capability7, nodes, racks, priority2, false);
        amClient.addContainerRequest(storedContainer1);
        amClient.addContainerRequest(storedContainer2);
        amClient.addContainerRequest(storedContainer3);
        amClient.addContainerRequest(storedContainer4);
        amClient.addContainerRequest(storedContainer5);
        amClient.addContainerRequest(storedContainer6);
        amClient.addContainerRequest(storedContainer7);
        // Add some CRs with allocReqIds... These will not be returned by
        // the default getMatchingRequests
        ContainerRequest storedContainer11 = new ContainerRequest(capability1, nodes, racks, priority, 1);
        ContainerRequest storedContainer33 = new ContainerRequest(capability3, nodes, racks, priority, 3);
        ContainerRequest storedContainer43 = new ContainerRequest(capability4, nodes, racks, priority, 3);
        amClient.addContainerRequest(storedContainer11);
        amClient.addContainerRequest(storedContainer33);
        amClient.addContainerRequest(storedContainer43);
        // test matching of containers
        List<? extends Collection<ContainerRequest>> matches;
        ContainerRequest storedRequest;
        // exact match
        Resource testCapability1 = Resource.newInstance(1024, 2);
        matches = amClient.getMatchingRequests(priority, node, testCapability1);
        verifyMatches(matches, 1);
        storedRequest = matches.get(0).iterator().next();
        assertEquals(storedContainer1, storedRequest);
        amClient.removeContainerRequest(storedContainer1);
        // exact match for allocReqId 1
        Collection<ContainerRequest> reqIdMatches = amClient.getMatchingRequests(1);
        assertEquals(1, reqIdMatches.size());
        storedRequest = reqIdMatches.iterator().next();
        assertEquals(storedContainer11, storedRequest);
        amClient.removeContainerRequest(storedContainer11);
        // exact match for allocReqId 3
        reqIdMatches = amClient.getMatchingRequests(3);
        assertEquals(2, reqIdMatches.size());
        Iterator<ContainerRequest> iter = reqIdMatches.iterator();
        storedRequest = iter.next();
        assertEquals(storedContainer43, storedRequest);
        amClient.removeContainerRequest(storedContainer43);
        storedRequest = iter.next();
        assertEquals(storedContainer33, storedRequest);
        amClient.removeContainerRequest(storedContainer33);
        // exact matching with order maintained
        Resource testCapability2 = Resource.newInstance(2000, 1);
        matches = amClient.getMatchingRequests(priority, node, testCapability2);
        verifyMatches(matches, 2);
        // must be returned in the order they were made
        int i = 0;
        for (ContainerRequest storedRequest1 : matches.get(0)) {
            if (i++ == 0) {
                assertEquals(storedContainer4, storedRequest1);
            } else {
                assertEquals(storedContainer6, storedRequest1);
            }
        }
        amClient.removeContainerRequest(storedContainer6);
        // matching with larger container. all requests returned
        Resource testCapability3 = Resource.newInstance(4000, 4);
        matches = amClient.getMatchingRequests(priority, node, testCapability3);
        assert (matches.size() == 4);
        Resource testCapability4 = Resource.newInstance(1024, 2);
        matches = amClient.getMatchingRequests(priority, node, testCapability4);
        assert (matches.size() == 2);
        // verify non-fitting containers are not returned and fitting ones are
        for (Collection<ContainerRequest> testSet : matches) {
            assertEquals(1, testSet.size());
            ContainerRequest testRequest = testSet.iterator().next();
            assertTrue(testRequest != storedContainer4);
            assertTrue(testRequest != storedContainer5);
            assert (testRequest == storedContainer2 || testRequest == storedContainer3);
        }
        Resource testCapability5 = Resource.newInstance(512, 4);
        matches = amClient.getMatchingRequests(priority, node, testCapability5);
        assert (matches.size() == 0);
        // verify requests without relaxed locality are only returned at specific
        // locations
        Resource testCapability7 = Resource.newInstance(2000, 1);
        matches = amClient.getMatchingRequests(priority2, ResourceRequest.ANY, testCapability7);
        assert (matches.size() == 0);
        matches = amClient.getMatchingRequests(priority2, node, testCapability7);
        assert (matches.size() == 1);
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 5 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAMRMClientMatchStorage.

//(timeout=60000)
@Test
public void testAMRMClientMatchStorage() throws YarnException, IOException {
    AMRMClientImpl<ContainerRequest> amClient = null;
    try {
        // start am rm client
        amClient = (AMRMClientImpl<ContainerRequest>) AMRMClient.<ContainerRequest>createAMRMClient();
        amClient.init(conf);
        amClient.start();
        amClient.registerApplicationMaster("Host", 10000, "");
        Priority priority1 = Records.newRecord(Priority.class);
        priority1.setPriority(2);
        ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, racks, priority);
        ContainerRequest storedContainer2 = new ContainerRequest(capability, nodes, racks, priority);
        ContainerRequest storedContainer3 = new ContainerRequest(capability, null, null, priority1);
        amClient.addContainerRequest(storedContainer1);
        amClient.addContainerRequest(storedContainer2);
        amClient.addContainerRequest(storedContainer3);
        // test addition and storage
        RemoteRequestsTable<ContainerRequest> remoteRequestsTable = amClient.getTable(0);
        int containersRequestedAny = remoteRequestsTable.get(priority, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        assertEquals(2, containersRequestedAny);
        containersRequestedAny = remoteRequestsTable.get(priority1, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        assertEquals(1, containersRequestedAny);
        List<? extends Collection<ContainerRequest>> matches = amClient.getMatchingRequests(priority, node, capability);
        verifyMatches(matches, 2);
        matches = amClient.getMatchingRequests(priority, rack, capability);
        verifyMatches(matches, 2);
        matches = amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability);
        verifyMatches(matches, 2);
        matches = amClient.getMatchingRequests(priority1, rack, capability);
        assertTrue(matches.isEmpty());
        matches = amClient.getMatchingRequests(priority1, ResourceRequest.ANY, capability);
        verifyMatches(matches, 1);
        // test removal
        amClient.removeContainerRequest(storedContainer3);
        matches = amClient.getMatchingRequests(priority, node, capability);
        verifyMatches(matches, 2);
        amClient.removeContainerRequest(storedContainer2);
        matches = amClient.getMatchingRequests(priority, node, capability);
        verifyMatches(matches, 1);
        matches = amClient.getMatchingRequests(priority, rack, capability);
        verifyMatches(matches, 1);
        // test matching of containers
        ContainerRequest storedRequest = matches.get(0).iterator().next();
        assertEquals(storedContainer1, storedRequest);
        amClient.removeContainerRequest(storedContainer1);
        matches = amClient.getMatchingRequests(priority, ResourceRequest.ANY, capability);
        assertTrue(matches.isEmpty());
        matches = amClient.getMatchingRequests(priority1, ResourceRequest.ANY, capability);
        assertTrue(matches.isEmpty());
        // 0 requests left. everything got cleaned up
        assertTrue(amClient.getTable(0).isEmpty());
        // go through an exemplary allocation, matching and release cycle
        amClient.addContainerRequest(storedContainer1);
        amClient.addContainerRequest(storedContainer3);
        // RM should allocate container within 2 calls to allocate()
        int allocatedContainerCount = 0;
        int iterationsLeft = 3;
        while (allocatedContainerCount < 2 && iterationsLeft-- > 0) {
            Log.getLog().info("Allocated " + allocatedContainerCount + " containers" + " with " + iterationsLeft + " iterations left");
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            assertEquals(0, amClient.ask.size());
            assertEquals(0, amClient.release.size());
            assertEquals(nodeCount, amClient.getClusterNodeCount());
            allocatedContainerCount += allocResponse.getAllocatedContainers().size();
            for (Container container : allocResponse.getAllocatedContainers()) {
                ContainerRequest expectedRequest = container.getPriority().equals(storedContainer1.getPriority()) ? storedContainer1 : storedContainer3;
                matches = amClient.getMatchingRequests(container.getPriority(), ResourceRequest.ANY, container.getResource());
                // test correct matched container is returned
                verifyMatches(matches, 1);
                ContainerRequest matchedRequest = matches.get(0).iterator().next();
                assertEquals(matchedRequest, expectedRequest);
                amClient.removeContainerRequest(matchedRequest);
                // assign this container, use it and release it
                amClient.releaseAssignedContainer(container.getId());
            }
            if (allocatedContainerCount < containersRequestedAny) {
                // let NM heartbeat to RM and trigger allocations
                triggerSchedulingWithNMHeartBeat();
            }
        }
        assertEquals(2, allocatedContainerCount);
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.release.size());
        assertEquals(0, amClient.ask.size());
        assertEquals(0, allocResponse.getAllocatedContainers().size());
        // 0 requests left. everything got cleaned up
        assertTrue(remoteRequestsTable.isEmpty());
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}