Examples with AllocateResponse org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse used on opensource projects

Example 71 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClientOnRMRestart method testAMRMClientResendsRequestsOnRMRestart.

// Test does major 6 steps verification.
// Step-1 : AMRMClient send allocate request for 3 container requests
// Step-2 : 3 containers are allocated by RM.
// Step-3 : AM Send 1 containerRequest(cRequest4) and 1 releaseRequests to
// RM
// Step-3.5 : AM Send 1 container resource increase request to RM
// Step-4 : On RM restart, AM(does not know RM is restarted) sends additional
// containerRequest(cRequest5) and blacklisted nodes.
// Intern RM send resync command
// Verify AM can recover increase request after resync
// Step-5 : Allocater after resync command & new containerRequest(cRequest6)
// Step-6 : RM allocates containers i.e cRequest4,cRequest5 and cRequest6
@Test(timeout = 60000)
public void testAMRMClientResendsRequestsOnRMRestart() throws Exception {
    UserGroupInformation.setLoginUser(null);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // Phase-1 Start 1st RM
    MyResourceManager rm1 = new MyResourceManager(conf, memStore);
    rm1.start();
    DrainDispatcher dispatcher = (DrainDispatcher) rm1.getRMContext().getDispatcher();
    // Submit the application
    RMApp app = rm1.submitApp(1024);
    dispatcher.await();
    MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // Node heartbeat
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId();
    rm1.sendAMLaunched(appAttemptId);
    dispatcher.await();
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId).getAMRMToken();
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    // Step-1 : AMRMClient send allocate request for 3 ContainerRequest
    // cRequest1 = h1, cRequest2 = h1,h2 and cRequest3 = h1
    // blacklisted nodes = h2
    AMRMClient<ContainerRequest> amClient = new MyAMRMClientImpl(rm1);
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    ContainerRequest cRequest1 = createReq(1, 1024, new String[] { "h1" });
    amClient.addContainerRequest(cRequest1);
    ContainerRequest cRequest2 = createReq(1, 1024, new String[] { "h1", "h2" });
    amClient.addContainerRequest(cRequest2);
    ContainerRequest cRequest3 = createReq(1, 1024, new String[] { "h1" });
    amClient.addContainerRequest(cRequest3);
    List<String> blacklistAdditions = new ArrayList<String>();
    List<String> blacklistRemoval = new ArrayList<String>();
    blacklistAdditions.add("h2");
    blacklistRemoval.add("h10");
    amClient.updateBlacklist(blacklistAdditions, blacklistRemoval);
    // remove from local list
    blacklistAdditions.remove("h2");
    AllocateResponse allocateResponse = amClient.allocate(0.1f);
    dispatcher.await();
    Assert.assertEquals("No of assignments must be 0", 0, allocateResponse.getAllocatedContainers().size());
    // Why 4 ask, why not 3 ask even h2 is blacklisted?
    // On blacklisting host,applicationmaster has to remove ask request from
    // remoterequest table.Here,test does not remove explicitely
    assertAsksAndReleases(4, 0, rm1);
    assertBlacklistAdditionsAndRemovals(1, 1, rm1);
    // Step-2 : NM heart beat is sent.
    // On 2nd AM allocate request, RM allocates 3 containers to AM
    // Node heartbeat
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    allocateResponse = amClient.allocate(0.2f);
    dispatcher.await();
    // 3 containers are allocated i.e for cRequest1, cRequest2 and cRequest3.
    Assert.assertEquals("No of assignments must be 0", 3, allocateResponse.getAllocatedContainers().size());
    assertAsksAndReleases(0, 0, rm1);
    assertBlacklistAdditionsAndRemovals(0, 0, rm1);
    List<Container> allocatedContainers = allocateResponse.getAllocatedContainers();
    // removed allocated container requests
    amClient.removeContainerRequest(cRequest1);
    amClient.removeContainerRequest(cRequest2);
    amClient.removeContainerRequest(cRequest3);
    allocateResponse = amClient.allocate(0.2f);
    dispatcher.await();
    Assert.assertEquals("No of assignments must be 0", 0, allocateResponse.getAllocatedContainers().size());
    assertAsksAndReleases(4, 0, rm1);
    assertBlacklistAdditionsAndRemovals(0, 0, rm1);
    // Step-3 : Send 1 containerRequest and 1 releaseRequests to RM
    ContainerRequest cRequest4 = createReq(1, 1024, new String[] { "h1" });
    amClient.addContainerRequest(cRequest4);
    int pendingRelease = 0;
    Iterator<Container> it = allocatedContainers.iterator();
    while (it.hasNext()) {
        amClient.releaseAssignedContainer(it.next().getId());
        pendingRelease++;
        it.remove();
        // remove one container
        break;
    }
    // Step-3.5 : Send 1 container resource increase request to RM
    Container container = it.next();
    ContainerId containerId = container.getId();
    // Make sure that container is in RUNNING state before sending increase
    // request
    nm1.nodeHeartbeat(containerId.getApplicationAttemptId(), containerId.getContainerId(), ContainerState.RUNNING);
    dispatcher.await();
    amClient.requestContainerUpdate(container, UpdateContainerRequest.newInstance(container.getVersion(), container.getId(), ContainerUpdateType.INCREASE_RESOURCE, Resource.newInstance(2048, 1), null));
    it.remove();
    allocateResponse = amClient.allocate(0.3f);
    dispatcher.await();
    Assert.assertEquals("No of assignments must be 0", 0, allocateResponse.getAllocatedContainers().size());
    assertAsksAndReleases(3, pendingRelease, rm1);
    // Verify there is one increase and zero decrease
    assertChanges(1, 0, rm1);
    assertBlacklistAdditionsAndRemovals(0, 0, rm1);
    int completedContainer = allocateResponse.getCompletedContainersStatuses().size();
    pendingRelease -= completedContainer;
    // Phase-2 start 2nd RM is up
    MyResourceManager rm2 = new MyResourceManager(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    ((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
    dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
    // NM should be rebooted on heartbeat, even first heartbeat for nm2
    NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
    Assert.assertEquals(NodeAction.RESYNC, hbResponse.getNodeAction());
    // new NM to represent NM re-register
    nm1 = new MockNM("h1:1234", 10240, rm2.getResourceTrackerService());
    NMContainerStatus containerReport = NMContainerStatus.newInstance(containerId, 0, ContainerState.RUNNING, Resource.newInstance(1024, 1), "recover container", 0, Priority.newInstance(0), 0);
    nm1.registerNode(Collections.singletonList(containerReport), Collections.singletonList(containerId.getApplicationAttemptId().getApplicationId()));
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    blacklistAdditions.add("h3");
    amClient.updateBlacklist(blacklistAdditions, null);
    blacklistAdditions.remove("h3");
    it = allocatedContainers.iterator();
    while (it.hasNext()) {
        amClient.releaseAssignedContainer(it.next().getId());
        pendingRelease++;
        it.remove();
    }
    ContainerRequest cRequest5 = createReq(1, 1024, new String[] { "h1", "h2" });
    amClient.addContainerRequest(cRequest5);
    // Step-4 : On RM restart, AM(does not know RM is restarted) sends
    // additional
    // containerRequest and blacklisted nodes.
    // Intern RM send resync command,AMRMClient resend allocate request
    allocateResponse = amClient.allocate(0.3f);
    dispatcher.await();
    completedContainer = allocateResponse.getCompletedContainersStatuses().size();
    pendingRelease -= completedContainer;
    assertAsksAndReleases(4, pendingRelease, rm2);
    // Verify there is one increase and zero decrease
    assertChanges(1, 0, rm2);
    assertBlacklistAdditionsAndRemovals(2, 0, rm2);
    ContainerRequest cRequest6 = createReq(1, 1024, new String[] { "h1", "h2", "h3" });
    amClient.addContainerRequest(cRequest6);
    // Step-5 : Allocater after resync command
    allocateResponse = amClient.allocate(0.5f);
    dispatcher.await();
    Assert.assertEquals("No of assignments must be 0", 0, allocateResponse.getAllocatedContainers().size());
    assertAsksAndReleases(5, 0, rm2);
    // Verify there is no increase or decrease requests any more
    assertChanges(0, 0, rm2);
    assertBlacklistAdditionsAndRemovals(0, 0, rm2);
    int noAssignedContainer = 0;
    int count = 5;
    while (count-- > 0) {
        nm1.nodeHeartbeat(true);
        dispatcher.await();
        allocateResponse = amClient.allocate(0.5f);
        dispatcher.await();
        noAssignedContainer += allocateResponse.getAllocatedContainers().size();
        if (noAssignedContainer == 3) {
            break;
        }
        Thread.sleep(1000);
    }
    // Step-6 : RM allocates containers i.e cRequest4,cRequest5 and cRequest6
    Assert.assertEquals("Number of container should be 3", 3, noAssignedContainer);
    amClient.stop();
    rm1.stop();
    rm2.stop();
}

Example 72 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyE2E.

/*
   * This test validates register, allocate and finish of an application through
   * the AMRMPRoxy.
   */
@Test(timeout = 120000)
public void testAMRMProxyE2E() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testAMRMProxyE2E", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        // the client has to connect to AMRMProxy
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit application
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        LOG.info("testAMRMProxyE2E - Register Application Master");
        RegisterApplicationMasterResponse responseRegister = client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        Assert.assertNotNull(responseRegister);
        Assert.assertNotNull(responseRegister.getQueue());
        Assert.assertNotNull(responseRegister.getApplicationACLs());
        Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
        Assert.assertNotNull(responseRegister.getContainersFromPreviousAttempts());
        Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
        Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
        RMApp rmApp = cluster.getResourceManager().getRMContext().getRMApps().get(appId);
        Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
        LOG.info("testAMRMProxyE2E - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        AllocateResponse allocResponse = client.allocate(request);
        Assert.assertNotNull(allocResponse);
        Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
        request.setAskList(new ArrayList<ResourceRequest>());
        request.setResponseId(request.getResponseId() + 1);
        Thread.sleep(1000);
        // RM should allocate container within 2 calls to allocate()
        allocResponse = client.allocate(request);
        Assert.assertNotNull(allocResponse);
        Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
        LOG.info("testAMRMPRoxy - Finish Application Master");
        FinishApplicationMasterResponse responseFinish = client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
        Assert.assertNotNull(responseFinish);
        Thread.sleep(500);
        Assert.assertNotEquals(RMAppState.FINISHED, rmApp.getState());
    }
}

Example 73 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyTokenRenewal.

/*
   * This test validates the token renewal from the AMRMPRoxy. The test verifies
   * that the received token from AMRMProxy is different from the previous one
   * within 5 requests.
   */
@Test(timeout = 120000)
public void testAMRMProxyTokenRenewal() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 1500);
        // RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS should be at least
        // RM_AM_EXPIRY_INTERVAL_MS * 1.5 *3
        conf.setInt(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 6);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        LOG.info("testAMRMProxyTokenRenewal - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        Token lastToken = null;
        AllocateResponse response = null;
        for (int i = 0; i < 5; i++) {
            response = client.allocate(request);
            request.setResponseId(request.getResponseId() + 1);
            if (response.getAMRMToken() != null && !response.getAMRMToken().equals(lastToken)) {
                break;
            }
            lastToken = response.getAMRMToken();
            // Time slot to be sure the AMRMProxy renew the token
            Thread.sleep(1500);
        }
        Assert.assertFalse(response.getAMRMToken().equals(lastToken));
        LOG.info("testAMRMPRoxy - Finish Application Master");
        client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    }
}

Example 74 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClientAsync method createAllocateResponse.

private AllocateResponse createAllocateResponse(List<ContainerStatus> completed, List<Container> allocated, List<Container> increased, List<Container> decreased, List<NMToken> nmTokens) {
    List<UpdatedContainer> updatedContainers = new ArrayList<>();
    for (Container c : increased) {
        updatedContainers.add(UpdatedContainer.newInstance(ContainerUpdateType.INCREASE_RESOURCE, c));
    }
    for (Container c : decreased) {
        updatedContainers.add(UpdatedContainer.newInstance(ContainerUpdateType.DECREASE_RESOURCE, c));
    }
    AllocateResponse response = AllocateResponse.newInstance(0, completed, allocated, new ArrayList<NodeReport>(), null, null, 1, null, nmTokens, updatedContainers);
    return response;
}

Example 75 with AllocateResponse

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse in project hadoop by apache.

the class TestAMRMClientAsync method testCallAMRMClientAsyncStopFromCallbackHandler.

@Test(timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandler() throws YarnException, IOException, InterruptedException {
    Configuration conf = new Configuration();
    TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
    @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
    List<ContainerStatus> completed = Arrays.asList(ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0));
    final AllocateResponse response = createAllocateResponse(completed, new ArrayList<Container>(), null);
    when(client.allocate(anyFloat())).thenReturn(response);
    AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
    callbackHandler.asynClient = asyncClient;
    asyncClient.init(conf);
    asyncClient.start();
    synchronized (callbackHandler.notifier) {
        asyncClient.registerApplicationMaster("localhost", 1234, null);
        while (callbackHandler.notify == false) {
            try {
                callbackHandler.notifier.wait();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}