Examples with AllocateRequest org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest used on opensource projects

Example 11 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMProxyService method getContainersAndAssert.

private List<Container> getContainersAndAssert(int appId, int numberOfResourceRequests) throws Exception {
    AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
    allocateRequest.setResponseId(1);
    List<Container> containers = new ArrayList<Container>(numberOfResourceRequests);
    List<ResourceRequest> askList = new ArrayList<ResourceRequest>(numberOfResourceRequests);
    for (int testAppId = 0; testAppId < numberOfResourceRequests; testAppId++) {
        askList.add(createResourceRequest("test-node-" + Integer.toString(testAppId), 6000, 2, testAppId % 5, 1));
    }
    allocateRequest.setAskList(askList);
    AllocateResponse allocateResponse = allocate(appId, allocateRequest);
    Assert.assertNotNull("allocate() returned null response", allocateResponse);
    Assert.assertNull("new AMRMToken from RM should have been nulled by AMRMProxyService", allocateResponse.getAMRMToken());
    containers.addAll(allocateResponse.getAllocatedContainers());
    // Send max 10 heart beats to receive all the containers. If not, we will
    // fail the test
    int numHeartbeat = 0;
    while (containers.size() < askList.size() && numHeartbeat++ < 10) {
        allocateResponse = allocate(appId, Records.newRecord(AllocateRequest.class));
        Assert.assertNotNull("allocate() returned null response", allocateResponse);
        Assert.assertNull("new AMRMToken from RM should have been nulled by AMRMProxyService", allocateResponse.getAMRMToken());
        containers.addAll(allocateResponse.getAllocatedContainers());
        LOG.info("Number of allocated containers in this request: " + Integer.toString(allocateResponse.getAllocatedContainers().size()));
        LOG.info("Total number of allocated containers: " + Integer.toString(containers.size()));
        Thread.sleep(10);
    }
    // We broadcast the request, the number of containers we received will be
    // higher than we ask
    Assert.assertTrue("The asklist count is not same as response", askList.size() <= containers.size());
    return containers;
}

Example 12 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMRPCNodeUpdates method testAMRMUnusableNodes.

@Test
public void testAMRMUnusableNodes() throws Exception {
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10000);
    MockNM nm2 = rm.registerNode("127.0.0.2:1234", 10000);
    MockNM nm3 = rm.registerNode("127.0.0.3:1234", 10000);
    MockNM nm4 = rm.registerNode("127.0.0.4:1234", 10000);
    dispatcher.await();
    RMApp app1 = rm.submitApp(2000);
    // Trigger the scheduling so the AM gets 'launched' on nm1
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    // register AM returns no unusable node
    am1.registerAppAttempt();
    // allocate request returns no updated node
    AllocateRequest allocateRequest1 = AllocateRequest.newInstance(0, 0F, null, null, null);
    AllocateResponse response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    List<NodeReport> updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(0, updatedNodes.size());
    syncNodeHeartbeat(nm4, false);
    // allocate request returns updated node
    allocateRequest1 = AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null, null);
    response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    NodeReport nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState());
    // resending the allocate request returns the same result
    response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState());
    syncNodeLost(nm3);
    // subsequent allocate request returns delta
    allocateRequest1 = AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null, null);
    response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm3.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.LOST, nr.getNodeState());
    // registering another AM gives it the complete failed list
    RMApp app2 = rm.submitApp(2000);
    // Trigger nm2 heartbeat so that AM gets launched on it
    nm2.nodeHeartbeat(true);
    RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
    MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId());
    // register AM returns all unusable nodes
    am2.registerAppAttempt();
    // allocate request returns no updated node
    AllocateRequest allocateRequest2 = AllocateRequest.newInstance(0, 0F, null, null, null);
    AllocateResponse response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2);
    updatedNodes = response2.getUpdatedNodes();
    Assert.assertEquals(0, updatedNodes.size());
    syncNodeHeartbeat(nm4, true);
    // both AM's should get delta updated nodes
    allocateRequest1 = AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null, null);
    response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
    updatedNodes = response1.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());
    allocateRequest2 = AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null, null);
    response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2);
    updatedNodes = response2.getUpdatedNodes();
    Assert.assertEquals(1, updatedNodes.size());
    nr = updatedNodes.iterator().next();
    Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
    Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());
    // subsequent allocate calls should return no updated nodes
    allocateRequest2 = AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null, null);
    response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2);
    updatedNodes = response2.getUpdatedNodes();
    Assert.assertEquals(0, updatedNodes.size());
// how to do the above for LOST node
}

Example 13 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMRPCResponseId method testARRMResponseId.

@Test
public void testARRMResponseId() throws Exception {
    MockNM nm1 = rm.registerNode("h1:1234", 5000);
    RMApp app = rm.submitApp(2000);
    // Trigger the scheduling so the AM gets 'launched'
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
    am.registerAppAttempt();
    AllocateRequest allocateRequest = AllocateRequest.newInstance(0, 0F, null, null, null);
    AllocateResponse response = allocate(attempt.getAppAttemptId(), allocateRequest);
    Assert.assertEquals(1, response.getResponseId());
    Assert.assertTrue(response.getAMCommand() == null);
    allocateRequest = AllocateRequest.newInstance(response.getResponseId(), 0F, null, null, null);
    response = allocate(attempt.getAppAttemptId(), allocateRequest);
    Assert.assertEquals(2, response.getResponseId());
    /* try resending */
    response = allocate(attempt.getAppAttemptId(), allocateRequest);
    Assert.assertEquals(2, response.getResponseId());
    /** try sending old request again **/
    allocateRequest = AllocateRequest.newInstance(0, 0F, null, null, null);
    try {
        allocate(attempt.getAppAttemptId(), allocateRequest);
        Assert.fail();
    } catch (Exception e) {
        Assert.assertTrue(e.getCause() instanceof InvalidApplicationMasterRequestException);
    }
}

Example 14 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMTokens method testMasterKeyRollOver.

/**
   * Validate master-key-roll-over and that tokens are usable even after
   * master-key-roll-over.
   * 
   * @throws Exception
   */
@Test
public void testMasterKeyRollOver() throws Exception {
    conf.setLong(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec);
    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:0");
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    Long startTime = System.currentTimeMillis();
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    ApplicationMasterProtocol rmClient = null;
    AMRMTokenSecretManager appTokenSecretManager = rm.getRMContext().getAMRMTokenSecretManager();
    MasterKeyData oldKey = appTokenSecretManager.getMasterKey();
    Assert.assertNotNull(oldKey);
    try {
        MockNM nm1 = rm.registerNode("localhost:1234", 5120);
        RMApp app = rm.submitApp(1024);
        nm1.nodeHeartbeat(true);
        int waitCount = 0;
        while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) {
            LOG.info("Waiting for AM Launch to happen..");
            Thread.sleep(1000);
        }
        Assert.assertNotNull(containerManager.containerTokens);
        RMAppAttempt attempt = app.getCurrentAppAttempt();
        ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
        // Create a client to the RM.
        UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
        Credentials credentials = containerManager.getContainerCredentials();
        final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
        Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
        currentUser.addToken(amRMToken);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
        rmClient.registerApplicationMaster(request);
        // One allocate call.
        AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        // At mean time, the old AMRMToken should continue to work
        while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
            rmClient.allocate(allocateRequest);
            Thread.sleep(500);
        }
        MasterKeyData newKey = appTokenSecretManager.getMasterKey();
        Assert.assertNotNull(newKey);
        Assert.assertFalse("Master key should have changed!", oldKey.equals(newKey));
        // Another allocate call with old AMRMToken. Should continue to work.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        waitCount = 0;
        while (waitCount++ <= maxWaitAttempts) {
            if (appTokenSecretManager.getCurrnetMasterKeyData() != oldKey) {
                break;
            }
            try {
                rmClient.allocate(allocateRequest);
            } catch (Exception ex) {
                break;
            }
            Thread.sleep(200);
        }
        // active the nextMasterKey, and replace the currentMasterKey
        Assert.assertTrue(appTokenSecretManager.getCurrnetMasterKeyData().equals(newKey));
        Assert.assertTrue(appTokenSecretManager.getMasterKey().equals(newKey));
        Assert.assertTrue(appTokenSecretManager.getNextMasterKeyData() == null);
        // Create a new Token
        Token<AMRMTokenIdentifier> newToken = appTokenSecretManager.createAndGetAMRMToken(applicationAttemptId);
        SecurityUtil.setTokenService(newToken, rmBindAddress);
        currentUser.addToken(newToken);
        // Another allocate call. Should continue to work.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        rmClient = createRMClient(rm, conf, rpc, currentUser);
        allocateRequest = Records.newRecord(AllocateRequest.class);
        Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
        // Should not work by using the old AMRMToken.
        // To avoid using cached client
        rpc.stopProxy(rmClient, conf);
        try {
            currentUser.addToken(amRMToken);
            rmClient = createRMClient(rm, conf, rpc, currentUser);
            allocateRequest = Records.newRecord(AllocateRequest.class);
            Assert.assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
            Assert.fail("The old Token should not work");
        } catch (Exception ex) {
        // expect exception
        }
    } finally {
        rm.stop();
        if (rmClient != null) {
            // To avoid using cached client
            rpc.stopProxy(rmClient, conf);
        }
    }
}

Example 15 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestSchedulerUtils method testValidateResourceBlacklistRequest.

@Test
public void testValidateResourceBlacklistRequest() throws Exception {
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(new YarnConfiguration(), containerManager);
    rm.start();
    MockNM nm1 = rm.registerNode("localhost:1234", 5120);
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(2);
    acls.put(ApplicationAccessType.VIEW_APP, "*");
    RMApp app = rm.submitApp(1024, "appname", "appuser", acls);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
    waitForLaunchedState(attempt);
    // Create a client to the RM.
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
    Credentials credentials = containerManager.getContainerCredentials();
    final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
    Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);
    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
    });
    RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
    client.registerApplicationMaster(request);
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(Collections.singletonList(ResourceRequest.ANY), null);
    AllocateRequest allocateRequest = AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
    boolean error = false;
    try {
        client.allocate(allocateRequest);
    } catch (InvalidResourceBlacklistRequestException e) {
        error = true;
    }
    rm.stop();
    Assert.assertTrue("Didn't not catch InvalidResourceBlacklistRequestException", error);
}