Examples with Container org.apache.hadoop.yarn.api.records.Container used on opensource projects

Example 66 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestAMRMClientAsync method testAMRMClientAsyncShutDown.

@Test(timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
    Configuration conf = new Configuration();
    TestCallbackHandler callbackHandler = new TestCallbackHandler();
    @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
    createAllocateResponse(new ArrayList<ContainerStatus>(), new ArrayList<Container>(), null);
    when(client.allocate(anyFloat())).thenThrow(new ApplicationAttemptNotFoundException("app not found, shut down"));
    AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
    asyncClient.init(conf);
    asyncClient.start();
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    Thread.sleep(50);
    verify(client, times(1)).allocate(anyFloat());
    asyncClient.stop();
}

Example 67 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestAMRMClientAsync method runCallBackThrowOutException.

void runCallBackThrowOutException(TestCallbackHandler2 callbackHandler) throws InterruptedException, YarnException, IOException {
    Configuration conf = new Configuration();
    @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
    List<ContainerStatus> completed = Arrays.asList(ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0));
    final AllocateResponse response = createAllocateResponse(completed, new ArrayList<Container>(), null);
    when(client.allocate(anyFloat())).thenReturn(response);
    AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
    callbackHandler.asynClient = asyncClient;
    callbackHandler.throwOutException = true;
    asyncClient.init(conf);
    asyncClient.start();
    // call register and wait for error callback and stop
    synchronized (callbackHandler.notifier) {
        asyncClient.registerApplicationMaster("localhost", 1234, null);
        while (callbackHandler.notify == false) {
            try {
                callbackHandler.notifier.wait();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
    // verify error invoked
    verify(callbackHandler, times(0)).getProgress();
    verify(callbackHandler, times(1)).onError(any(Exception.class));
    // sleep to wait for a few heartbeat calls that can trigger callbacks
    Thread.sleep(50);
    // verify no more invocations after the first one.
    // ie. callback thread has stopped
    verify(callbackHandler, times(0)).getProgress();
    verify(callbackHandler, times(1)).onError(any(Exception.class));
}

Example 68 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestAMRMClientAsync method testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor.

@Test(timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor() throws YarnException, IOException, InterruptedException {
    Configuration conf = new Configuration();
    final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
    @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
    List<ContainerStatus> completed = Arrays.asList(ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0));
    final AllocateResponse response = createAllocateResponse(completed, new ArrayList<Container>(), null);
    when(client.allocate(anyFloat())).thenReturn(response);
    AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
    callbackHandler.asynClient = asyncClient;
    asyncClient.init(conf);
    asyncClient.start();
    Supplier<Boolean> checker = new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            return callbackHandler.notify;
        }
    };
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    asyncClient.waitFor(checker);
    Assert.assertTrue(checker.get());
}

Example 69 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestNMClientAsync method testNMClientAsync.

@Test(timeout = 10000)
public void testNMClientAsync() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(YarnConfiguration.NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE, 10);
    // Threads to run are more than the max size of the thread pool
    int expectedSuccess = 40;
    int expectedFailure = 40;
    asyncClient = new MockNMClientAsync1(expectedSuccess, expectedFailure);
    asyncClient.init(conf);
    Assert.assertEquals("The max thread pool size is not correctly set", 10, asyncClient.maxThreadPoolSize);
    asyncClient.start();
    for (int i = 0; i < expectedSuccess + expectedFailure; ++i) {
        if (i == expectedSuccess) {
            while (!((TestCallbackHandler1) asyncClient.getCallbackHandler()).isAllSuccessCallsExecuted()) {
                Thread.sleep(10);
            }
            asyncClient.setClient(mockNMClient(1));
        }
        Container container = mockContainer(i);
        ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        asyncClient.startContainerAsync(container, clc);
    }
    while (!((TestCallbackHandler1) asyncClient.getCallbackHandler()).isStartAndQueryFailureCallsExecuted()) {
        Thread.sleep(10);
    }
    asyncClient.setClient(mockNMClient(2));
    ((TestCallbackHandler1) asyncClient.getCallbackHandler()).path = false;
    for (int i = 0; i < expectedFailure; ++i) {
        Container container = mockContainer(expectedSuccess + expectedFailure + i);
        ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        asyncClient.startContainerAsync(container, clc);
    }
    while (!((TestCallbackHandler1) asyncClient.getCallbackHandler()).isIncreaseResourceFailureCallsExecuted()) {
        Thread.sleep(10);
    }
    while (!((TestCallbackHandler1) asyncClient.getCallbackHandler()).isStopFailureCallsExecuted()) {
        Thread.sleep(10);
    }
    for (String errorMsg : ((TestCallbackHandler1) asyncClient.getCallbackHandler()).errorMsgs) {
        System.out.println(errorMsg);
    }
    Assert.assertEquals("Error occurs in CallbackHandler", 0, ((TestCallbackHandler1) asyncClient.getCallbackHandler()).errorMsgs.size());
    for (String errorMsg : ((MockNMClientAsync1) asyncClient).errorMsgs) {
        System.out.println(errorMsg);
    }
    Assert.assertEquals("Error occurs in ContainerEventProcessor", 0, ((MockNMClientAsync1) asyncClient).errorMsgs.size());
    // may still not terminate and the containers may still not removed.
    while (asyncClient.containers.size() > 0) {
        Thread.sleep(10);
    }
    asyncClient.stop();
    Assert.assertFalse("The thread of Container Management Event Dispatcher is still alive", asyncClient.eventDispatcherThread.isAlive());
    Assert.assertTrue("The thread pool is not shut down", asyncClient.threadPool.isShutdown());
}

Example 70 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestOpportunisticContainerAllocation method testPromotionFromAcquired.

@Test(timeout = 60000)
public void testPromotionFromAcquired() throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
    int oppContainersRequestedAny = amClient.getTable(0).get(priority2, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC, capability).remoteRequest.getNumContainers();
    assertEquals(1, oppContainersRequestedAny);
    assertEquals(1, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    Map<ContainerId, Container> allocatedOpportContainers = new HashMap<>();
    int iterationsLeft = 50;
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<>();
    updateMetrics("Before Opp Allocation");
    while (allocatedContainerCount < oppContainersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        for (Container container : allocResponse.getAllocatedContainers()) {
            if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
                allocatedOpportContainers.put(container.getId(), container);
                removeCR(container);
            }
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < oppContainersRequestedAny) {
            // sleep to let NM's heartbeat to RM and trigger allocations
            sleep(100);
        }
    }
    assertEquals(oppContainersRequestedAny, allocatedContainerCount);
    assertEquals(oppContainersRequestedAny, allocatedOpportContainers.size());
    updateMetrics("After Opp Allocation / Before Promotion");
    try {
        Container c = allocatedOpportContainers.values().iterator().next();
        amClient.requestContainerUpdate(c, UpdateContainerRequest.newInstance(c.getVersion(), c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE, null, ExecutionType.OPPORTUNISTIC));
        Assert.fail("Should throw Exception..");
    } catch (IllegalArgumentException e) {
        System.out.println("## " + e.getMessage());
        Assert.assertTrue(e.getMessage().contains("target should be GUARANTEED and original should be OPPORTUNISTIC"));
    }
    Container c = allocatedOpportContainers.values().iterator().next();
    amClient.requestContainerUpdate(c, UpdateContainerRequest.newInstance(c.getVersion(), c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE, null, ExecutionType.GUARANTEED));
    iterationsLeft = 120;
    Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
    // do a few iterations to ensure RM is not going to send new containers
    while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
        // inform RM of rejection
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        // RM did not send new containers because AM does not need any
        if (allocResponse.getUpdatedContainers() != null) {
            for (UpdatedContainer updatedContainer : allocResponse.getUpdatedContainers()) {
                System.out.println("Got update..");
                updatedContainers.put(updatedContainer.getContainer().getId(), updatedContainer);
            }
        }
        if (iterationsLeft > 0) {
            // sleep to make sure NM's heartbeat
            sleep(100);
        }
    }
    updateMetrics("After Promotion");
    assertEquals(1, updatedContainers.size());
    for (ContainerId cId : allocatedOpportContainers.keySet()) {
        Container orig = allocatedOpportContainers.get(cId);
        UpdatedContainer updatedContainer = updatedContainers.get(cId);
        assertNotNull(updatedContainer);
        assertEquals(ExecutionType.GUARANTEED, updatedContainer.getContainer().getExecutionType());
        assertEquals(orig.getResource(), updatedContainer.getContainer().getResource());
        assertEquals(orig.getNodeId(), updatedContainer.getContainer().getNodeId());
        assertEquals(orig.getVersion() + 1, updatedContainer.getContainer().getVersion());
    }
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.ask.clear();
}