Examples with AllocateResponse org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse used on opensource projects

Example 76 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClientAsync method testAMRMClientAsync.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testAMRMClientAsync() throws Exception {
    Configuration conf = new Configuration();
    final AtomicBoolean heartbeatBlock = new AtomicBoolean(true);
    List<ContainerStatus> completed1 = Arrays.asList(ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0));
    List<Container> containers = Arrays.asList(Container.newInstance(null, null, null, null, null, null));
    final AllocateResponse response1 = createAllocateResponse(new ArrayList<ContainerStatus>(), containers, null);
    final AllocateResponse response2 = createAllocateResponse(completed1, new ArrayList<Container>(), null);
    final AllocateResponse response3 = createAllocateResponse(new ArrayList<ContainerStatus>(), new ArrayList<Container>(), containers, containers, null);
    final AllocateResponse emptyResponse = createAllocateResponse(new ArrayList<ContainerStatus>(), new ArrayList<Container>(), null);
    TestCallbackHandler callbackHandler = new TestCallbackHandler();
    final AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
    final AtomicInteger secondHeartbeatSync = new AtomicInteger(0);
    when(client.allocate(anyFloat())).thenReturn(response1).thenAnswer(new Answer<AllocateResponse>() {

        @Override
        public AllocateResponse answer(InvocationOnMock invocation) throws Throwable {
            secondHeartbeatSync.incrementAndGet();
            while (heartbeatBlock.get()) {
                synchronized (heartbeatBlock) {
                    heartbeatBlock.wait();
                }
            }
            secondHeartbeatSync.incrementAndGet();
            return response2;
        }
    }).thenReturn(response3).thenReturn(emptyResponse);
    when(client.registerApplicationMaster(anyString(), anyInt(), anyString())).thenReturn(null);
    when(client.getAvailableResources()).thenAnswer(new Answer<Resource>() {

        @Override
        public Resource answer(InvocationOnMock invocation) throws Throwable {
            // take client lock to simulate behavior of real impl
            synchronized (client) {
                Thread.sleep(10);
            }
            return null;
        }
    });
    AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
    asyncClient.init(conf);
    asyncClient.start();
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    // To test this, wait for the second heartbeat to be received. 
    while (secondHeartbeatSync.get() < 1) {
        Thread.sleep(10);
    }
    // time. Checks that heartbeat is not holding onto client lock
    assert (secondHeartbeatSync.get() < 2);
    asyncClient.getAvailableResources();
    // method returned. now unblock heartbeat
    assert (secondHeartbeatSync.get() < 2);
    synchronized (heartbeatBlock) {
        heartbeatBlock.set(false);
        heartbeatBlock.notifyAll();
    }
    // allocated containers should come before completed containers
    Assert.assertEquals(null, callbackHandler.takeCompletedContainers());
    // wait for the allocated containers from the first heartbeat's response
    while (callbackHandler.takeAllocatedContainers() == null) {
        Assert.assertEquals(null, callbackHandler.takeCompletedContainers());
        Thread.sleep(10);
    }
    // wait for the completed containers from the second heartbeat's response
    while (callbackHandler.takeCompletedContainers() == null) {
        Thread.sleep(10);
    }
    // wait for the changed containers from the thrid heartbeat's response
    while (callbackHandler.takeChangedContainers() == null) {
        Thread.sleep(10);
    }
    asyncClient.stop();
    Assert.assertEquals(null, callbackHandler.takeAllocatedContainers());
    Assert.assertEquals(null, callbackHandler.takeCompletedContainers());
    Assert.assertEquals(null, callbackHandler.takeChangedContainers());
}

Example 77 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClient method waitForContainerCompletion.

private void waitForContainerCompletion(int numIterations, AMRMClientImpl<ContainerRequest> amClient, Set<ContainerId> releases) throws YarnException, IOException {
    // do a few iterations to ensure RM is not going send new containers
    while (!releases.isEmpty() || numIterations-- > 0) {
        // inform RM of rejection
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        // RM did not send new containers because AM does not need any
        assertEquals(0, allocResponse.getAllocatedContainers().size());
        if (allocResponse.getCompletedContainersStatuses().size() > 0) {
            for (ContainerStatus cStatus : allocResponse.getCompletedContainersStatuses()) {
                if (releases.contains(cStatus.getContainerId())) {
                    assertEquals(cStatus.getState(), ContainerState.COMPLETE);
                    assertEquals(-100, cStatus.getExitStatus());
                    releases.remove(cStatus.getContainerId());
                }
            }
        }
        if (numIterations > 0) {
            // let NM heartbeat to RM and trigger allocations
            triggerSchedulingWithNMHeartBeat();
        }
    }
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
}

Example 78 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClient method testAllocRequestId.

private void testAllocRequestId(final AMRMClientImpl<ContainerRequest> amClient) throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 1));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 1));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 2));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 2));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 1));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority, 2));
    assertNumContainers(amClient, 0, 1, 1, 1, 9, 0);
    assertNumContainers(amClient, 1, 1, 1, 1, 9, 0);
    assertNumContainers(amClient, 2, 1, 1, 1, 9, 0);
    int containersRequestedAny = 3;
    // RM should allocate container within 2 calls to allocate()
    List<Container> allocatedContainers = new ArrayList<>();
    int iterationsLeft = 5;
    Set<ContainerId> releases = new TreeSet<ContainerId>();
    while (allocatedContainers.size() < containersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        allocatedContainers.addAll(allocResponse.getAllocatedContainers());
        for (Container container : allocResponse.getAllocatedContainers()) {
            ContainerId rejectContainerId = container.getId();
            releases.add(rejectContainerId);
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        if (allocatedContainers.size() < containersRequestedAny) {
            // let NM heartbeat to RM and trigger allocations
            triggerSchedulingWithNMHeartBeat();
        }
    }
    assertEquals(containersRequestedAny, allocatedContainers.size());
    Set<Long> expAllocIds = new HashSet<>(Arrays.asList(Long.valueOf(0), Long.valueOf(1), Long.valueOf(2)));
    Set<Long> actAllocIds = new HashSet<>();
    for (Container ac : allocatedContainers) {
        actAllocIds.add(Long.valueOf(ac.getAllocationRequestId()));
    }
    assertEquals(expAllocIds, actAllocIds);
    assertEquals(3, amClient.release.size());
    assertEquals(0, amClient.ask.size());
    waitForContainerCompletion(3, amClient, releases);
}

Example 79 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClient method getAllocatedContainersNumber.

private int getAllocatedContainersNumber(AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft) throws YarnException, IOException {
    int allocatedContainerCount = 0;
    while (iterationsLeft-- > 0) {
        Log.getLog().info("Allocated " + allocatedContainerCount + " containers" + " with " + iterationsLeft + " iterations left");
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        assertEquals(nodeCount, amClient.getClusterNodeCount());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        if (allocatedContainerCount == 0) {
            // let NM heartbeat to RM and trigger allocations
            triggerSchedulingWithNMHeartBeat();
        }
    }
    return allocatedContainerCount;
}

Example 80 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestApplicationMasterServiceProtocolOnHA method testAllocateOnHA.

@Test(timeout = 15000)
public void testAllocateOnHA() throws YarnException, IOException {
    AllocateRequest request = AllocateRequest.newInstance(0, 50f, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), ResourceBlacklistRequest.newInstance(new ArrayList<String>(), new ArrayList<String>()));
    AllocateResponse response = amClient.allocate(request);
    Assert.assertEquals(response, this.cluster.createFakeAllocateResponse());
}