Examples with ContainerRequest org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest used on opensource projects

Example 6 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAllocationWithBlacklist.

@Test(timeout = 60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
    AMRMClientImpl<ContainerRequest> amClient = null;
    try {
        // start am rm client
        amClient = (AMRMClientImpl<ContainerRequest>) AMRMClient.<ContainerRequest>createAMRMClient();
        amClient.init(conf);
        amClient.start();
        amClient.registerApplicationMaster("Host", 10000, "");
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, racks, priority);
        amClient.addContainerRequest(storedContainer1);
        assertEquals(3, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        List<String> localNodeBlacklist = new ArrayList<String>();
        localNodeBlacklist.add(node);
        // put node in black list, so no container assignment
        amClient.updateBlacklist(localNodeBlacklist, null);
        int allocatedContainerCount = getAllocatedContainersNumber(amClient, DEFAULT_ITERATION);
        // the only node is in blacklist, so no allocation
        assertEquals(0, allocatedContainerCount);
        // Remove node from blacklist, so get assigned with 2
        amClient.updateBlacklist(null, localNodeBlacklist);
        ContainerRequest storedContainer2 = new ContainerRequest(capability, nodes, racks, priority);
        amClient.addContainerRequest(storedContainer2);
        allocatedContainerCount = getAllocatedContainersNumber(amClient, DEFAULT_ITERATION);
        assertEquals(2, allocatedContainerCount);
        // Test in case exception in allocate(), blacklist is kept
        assertTrue(amClient.blacklistAdditions.isEmpty());
        assertTrue(amClient.blacklistRemovals.isEmpty());
        // create a invalid ContainerRequest - memory value is minus
        ContainerRequest invalidContainerRequest = new ContainerRequest(Resource.newInstance(-1024, 1), nodes, racks, priority);
        amClient.addContainerRequest(invalidContainerRequest);
        amClient.updateBlacklist(localNodeBlacklist, null);
        try {
            // allocate() should complain as ContainerRequest is invalid.
            amClient.allocate(0.1f);
            fail("there should be an exception here.");
        } catch (Exception e) {
            assertEquals(1, amClient.blacklistAdditions.size());
        }
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 7 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAMRMClientMatchingFitInferredRack.

@Test(timeout = 60000)
public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException {
    AMRMClientImpl<ContainerRequest> amClient = null;
    try {
        // start am rm client
        amClient = new AMRMClientImpl<ContainerRequest>();
        amClient.init(conf);
        amClient.start();
        amClient.registerApplicationMaster("Host", 10000, "");
        Resource capability = Resource.newInstance(1024, 2);
        ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, null, priority);
        amClient.addContainerRequest(storedContainer1);
        // verify matching with original node and inferred rack
        List<? extends Collection<ContainerRequest>> matches;
        ContainerRequest storedRequest;
        // exact match node
        matches = amClient.getMatchingRequests(priority, node, capability);
        verifyMatches(matches, 1);
        storedRequest = matches.get(0).iterator().next();
        assertEquals(storedContainer1, storedRequest);
        // inferred match rack
        matches = amClient.getMatchingRequests(priority, rack, capability);
        verifyMatches(matches, 1);
        storedRequest = matches.get(0).iterator().next();
        assertEquals(storedContainer1, storedRequest);
        // inferred rack match no longer valid after request is removed
        amClient.removeContainerRequest(storedContainer1);
        matches = amClient.getMatchingRequests(priority, rack, capability);
        assertTrue(matches.isEmpty());
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}

Example 8 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAllocation.

private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient) throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    assertNumContainers(amClient, 0, 2, 2, 2, 3, 0);
    int containersRequestedAny = 2;
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    int iterationsLeft = 3;
    Set<ContainerId> releases = new TreeSet<ContainerId>();
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
    while (allocatedContainerCount < containersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        assertEquals(nodeCount, amClient.getClusterNodeCount());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        for (Container container : allocResponse.getAllocatedContainers()) {
            ContainerId rejectContainerId = container.getId();
            releases.add(rejectContainerId);
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            if (receivedNMTokens.containsKey(nodeID)) {
                Assert.fail("Received token again for : " + nodeID);
            }
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < containersRequestedAny) {
            // let NM heartbeat to RM and trigger allocations
            triggerSchedulingWithNMHeartBeat();
        }
    }
    // Should receive atleast 1 token
    Assert.assertTrue(receivedNMTokens.size() > 0 && receivedNMTokens.size() <= nodeCount);
    assertEquals(allocatedContainerCount, containersRequestedAny);
    assertEquals(2, amClient.release.size());
    assertEquals(0, amClient.ask.size());
    // need to tell the AMRMClient that we dont need these resources anymore
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    assertEquals(3, amClient.ask.size());
    // send 0 container count request for resources that are no longer needed
    ResourceRequest snoopRequest = amClient.ask.iterator().next();
    assertEquals(0, snoopRequest.getNumContainers());
    // test RPC exception handling
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    snoopRequest = amClient.ask.iterator().next();
    assertEquals(2, snoopRequest.getNumContainers());
    ApplicationMasterProtocol realRM = amClient.rmClient;
    try {
        ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
        when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(new Answer<AllocateResponse>() {

            public AllocateResponse answer(InvocationOnMock invocation) throws Exception {
                amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
                amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
                throw new Exception();
            }
        });
        amClient.rmClient = mockRM;
        amClient.allocate(0.1f);
    } catch (Exception ioe) {
    } finally {
        amClient.rmClient = realRM;
    }
    assertEquals(2, amClient.release.size());
    assertEquals(3, amClient.ask.size());
    snoopRequest = amClient.ask.iterator().next();
    // verify that the remove request made in between makeRequest and allocate 
    // has not been lost
    assertEquals(0, snoopRequest.getNumContainers());
    waitForContainerCompletion(3, amClient, releases);
}

Example 9 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method doContainerResourceChange.

private void doContainerResourceChange(final AMRMClient<ContainerRequest> amClient, List<Container> containers) throws YarnException, IOException {
    Assert.assertEquals(3, containers.size());
    // remember the container IDs
    Container container1 = containers.get(0);
    Container container2 = containers.get(1);
    Container container3 = containers.get(2);
    AMRMClientImpl<ContainerRequest> amClientImpl = (AMRMClientImpl<ContainerRequest>) amClient;
    Assert.assertEquals(0, amClientImpl.change.size());
    // verify newer request overwrites older request for the container1
    amClientImpl.requestContainerUpdate(container1, UpdateContainerRequest.newInstance(container1.getVersion(), container1.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(2048, 1), null));
    amClientImpl.requestContainerUpdate(container1, UpdateContainerRequest.newInstance(container1.getVersion(), container1.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(4096, 1), null));
    Assert.assertEquals(Resource.newInstance(4096, 1), amClientImpl.change.get(container1.getId()).getValue().getCapability());
    // verify new decrease request cancels old increase request for container1
    amClientImpl.requestContainerUpdate(container1, UpdateContainerRequest.newInstance(container1.getVersion(), container1.getId(), ContainerUpdateType.DECREASE_RESOURCE, Resource.newInstance(512, 1), null));
    Assert.assertEquals(Resource.newInstance(512, 1), amClientImpl.change.get(container1.getId()).getValue().getCapability());
    // request resource increase for container2
    amClientImpl.requestContainerUpdate(container2, UpdateContainerRequest.newInstance(container2.getVersion(), container2.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(2048, 1), null));
    Assert.assertEquals(Resource.newInstance(2048, 1), amClientImpl.change.get(container2.getId()).getValue().getCapability());
    // verify release request will cancel pending change requests for the same
    // container
    amClientImpl.requestContainerUpdate(container3, UpdateContainerRequest.newInstance(container3.getVersion(), container3.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(2048, 1), null));
    Assert.assertEquals(3, amClientImpl.pendingChange.size());
    amClientImpl.releaseAssignedContainer(container3.getId());
    Assert.assertEquals(2, amClientImpl.pendingChange.size());
    // as of now: container1 asks to decrease to (512, 1)
    //            container2 asks to increase to (2048, 1)
    // send allocation requests
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    Assert.assertEquals(0, amClientImpl.change.size());
    // we should get decrease confirmation right away
    List<UpdatedContainer> updatedContainers = allocResponse.getUpdatedContainers();
    Assert.assertEquals(1, updatedContainers.size());
    // we should get increase allocation after the next NM's heartbeat to RM
    triggerSchedulingWithNMHeartBeat();
    // get allocations
    allocResponse = amClient.allocate(0.1f);
    updatedContainers = allocResponse.getUpdatedContainers();
    Assert.assertEquals(1, updatedContainers.size());
}

Example 10 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClient method testAskWithNodeLabels.

@Test(timeout = 30000)
public void testAskWithNodeLabels() {
    AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
    // add exp=x to ANY
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
    Assert.assertEquals(1, client.ask.size());
    Assert.assertEquals("x", client.ask.iterator().next().getNodeLabelExpression());
    // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "a"));
    Assert.assertEquals(1, client.ask.size());
    Assert.assertEquals("a", client.ask.iterator().next().getNodeLabelExpression());
    // add exp=x to ANY, rack and node, only resource request has ANY resource
    // name will be assigned the label expression
    // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "y"));
    Assert.assertEquals(1, client.ask.size());
    for (ResourceRequest req : client.ask) {
        if (ResourceRequest.ANY.equals(req.getResourceName())) {
            Assert.assertEquals("y", req.getNodeLabelExpression());
        } else {
            Assert.assertNull(req.getNodeLabelExpression());
        }
    }
    // set container with nodes and racks with labels
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), new String[] { "rack1" }, new String[] { "node1", "node2" }, Priority.UNDEFINED, true, "y"));
    for (ResourceRequest req : client.ask) {
        if (ResourceRequest.ANY.equals(req.getResourceName())) {
            Assert.assertEquals("y", req.getNodeLabelExpression());
        } else {
            Assert.assertNull(req.getNodeLabelExpression());
        }
    }
}