Examples with ContainerRequest org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest used on opensource projects

Example 16 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestNMClient method setup.

@Before
public void setup() throws YarnException, IOException {
    // start minicluster
    conf = new YarnConfiguration();
    yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
    yarnCluster.init(conf);
    yarnCluster.start();
    assertNotNull(yarnCluster);
    assertEquals(STATE.STARTED, yarnCluster.getServiceState());
    // start rm client
    yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();
    assertNotNull(yarnClient);
    assertEquals(STATE.STARTED, yarnClient.getServiceState());
    // get node info
    nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
    // submit new app
    ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    // set the application name
    appContext.setApplicationName("Test");
    // Set the priority for the application master
    Priority pri = Priority.newInstance(0);
    appContext.setPriority(pri);
    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("default");
    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    // unmanaged AM
    appContext.setUnmanagedAM(true);
    // Create the request to send to the applications manager
    SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class);
    appRequest.setApplicationSubmissionContext(appContext);
    // Submit the application to the applications manager
    yarnClient.submitApplication(appContext);
    // wait for app to start
    int iterationsLeft = 30;
    RMAppAttempt appAttempt = null;
    while (iterationsLeft > 0) {
        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
            attemptId = appReport.getCurrentApplicationAttemptId();
            appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps().get(attemptId.getApplicationId()).getCurrentAppAttempt();
            while (true) {
                if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
                    break;
                }
            }
            break;
        }
        sleep(1000);
        --iterationsLeft;
    }
    if (iterationsLeft == 0) {
        fail("Application hasn't bee started");
    }
    // Just dig into the ResourceManager and get the AMRMToken just for the sake
    // of testing.
    UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
    UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
    //creating an instance NMTokenCase
    nmTokenCache = new NMTokenCache();
    // start am rm client
    rmClient = (AMRMClientImpl<ContainerRequest>) AMRMClient.<ContainerRequest>createAMRMClient();
    //setting an instance NMTokenCase
    rmClient.setNMTokenCache(nmTokenCache);
    rmClient.init(conf);
    rmClient.start();
    assertNotNull(rmClient);
    assertEquals(STATE.STARTED, rmClient.getServiceState());
    // start am nm client
    nmClient = (NMClientImpl) NMClient.createNMClient();
    //propagating the AMRMClient NMTokenCache instance
    nmClient.setNMTokenCache(rmClient.getNMTokenCache());
    nmClient.init(conf);
    nmClient.start();
    assertNotNull(nmClient);
    assertEquals(STATE.STARTED, nmClient.getServiceState());
}

Example 17 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClientContainerRequest method testInvalidValidWhenOldRemoved.

@Test
public void testInvalidValidWhenOldRemoved() {
    AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
    Configuration conf = new Configuration();
    conf.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class);
    client.init(conf);
    Resource capability = Resource.newInstance(1024, 1);
    ContainerRequest request1 = new ContainerRequest(capability, new String[] { "host1", "host2" }, null, Priority.newInstance(1), false);
    client.addContainerRequest(request1);
    client.removeContainerRequest(request1);
    ContainerRequest request2 = new ContainerRequest(capability, new String[] { "host3" }, null, Priority.newInstance(1), true);
    client.addContainerRequest(request2);
    client.removeContainerRequest(request2);
    ContainerRequest request3 = new ContainerRequest(capability, new String[] { "host1", "host2" }, null, Priority.newInstance(1), false);
    client.addContainerRequest(request3);
    client.removeContainerRequest(request3);
    ContainerRequest request4 = new ContainerRequest(capability, null, new String[] { "rack1" }, Priority.newInstance(1), true);
    client.addContainerRequest(request4);
}

Example 18 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClientContainerRequest method testOpportunisticAndGuaranteedRequests.

@Test
public void testOpportunisticAndGuaranteedRequests() {
    AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
    Configuration conf = new Configuration();
    conf.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class);
    client.init(conf);
    Resource capability = Resource.newInstance(1024, 1);
    ContainerRequest request = new ContainerRequest(capability, new String[] { "host1", "host2" }, new String[] { "/rack2" }, Priority.newInstance(1));
    client.addContainerRequest(request);
    verifyResourceRequest(client, request, "host1", true);
    verifyResourceRequest(client, request, "host2", true);
    verifyResourceRequest(client, request, "/rack1", true);
    verifyResourceRequest(client, request, "/rack2", true);
    verifyResourceRequest(client, request, ResourceRequest.ANY, true);
    ContainerRequest request2 = new ContainerRequest(capability, new String[] { "host1", "host2" }, new String[] { "/rack2" }, Priority.newInstance(1), 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true));
    client.addContainerRequest(request2);
    verifyResourceRequest(client, request, "host1", true, ExecutionType.OPPORTUNISTIC);
    verifyResourceRequest(client, request, "host2", true, ExecutionType.OPPORTUNISTIC);
    verifyResourceRequest(client, request, "/rack1", true, ExecutionType.OPPORTUNISTIC);
    verifyResourceRequest(client, request, "/rack2", true, ExecutionType.OPPORTUNISTIC);
    verifyResourceRequest(client, request, ResourceRequest.ANY, true, ExecutionType.OPPORTUNISTIC);
}

Example 19 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClientContainerRequest method testFillInRacks.

@Test
public void testFillInRacks() {
    AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
    Configuration conf = new Configuration();
    conf.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class);
    client.init(conf);
    Resource capability = Resource.newInstance(1024, 1);
    ContainerRequest request = new ContainerRequest(capability, new String[] { "host1", "host2" }, new String[] { "/rack2" }, Priority.newInstance(1));
    client.addContainerRequest(request);
    verifyResourceRequest(client, request, "host1", true);
    verifyResourceRequest(client, request, "host2", true);
    verifyResourceRequest(client, request, "/rack1", true);
    verifyResourceRequest(client, request, "/rack2", true);
    verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}

Example 20 with ContainerRequest

use of org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest in project hadoop by apache.

the class TestAMRMClientOnRMRestart method testAMRMClientOnAMRMTokenRollOverOnRMRestart.

// Test verify for AM issued with rolled-over AMRMToken
// is still able to communicate with restarted RM.
@Test(timeout = 30000)
public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception {
    conf.setLong(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec);
    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start first RM
    MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore);
    rm1.start();
    DrainDispatcher dispatcher = (DrainDispatcher) rm1.getRMContext().getDispatcher();
    Long startTime = System.currentTimeMillis();
    // Submit the application
    RMApp app = rm1.submitApp(1024);
    dispatcher.await();
    MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // Node heartbeat
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId();
    rm1.sendAMLaunched(appAttemptId);
    dispatcher.await();
    AMRMTokenSecretManager amrmTokenSecretManagerForRM1 = rm1.getRMContext().getAMRMTokenSecretManager();
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    AMRMClient<ContainerRequest> amClient = new MyAMRMClientImpl(rm1);
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("h1", 10000, "");
    amClient.allocate(0.1f);
    // At mean time, the old AMRMToken should continue to work
    while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
        amClient.allocate(0.1f);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        // DO NOTHING
        }
    }
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getMasterKey().getMasterKey().getKeyId() != token.decodeIdentifier().getKeyId());
    amClient.allocate(0.1f);
    // active the nextMasterKey, and replace the currentMasterKey
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newToken = amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId);
    int waitCount = 0;
    while (waitCount++ <= 50) {
        if (amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData().getMasterKey().getKeyId() != token.decodeIdentifier().getKeyId()) {
            break;
        }
        try {
            amClient.allocate(0.1f);
        } catch (Exception ex) {
            break;
        }
        Thread.sleep(500);
    }
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getNextMasterKeyData() == null);
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData().getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
    // start 2nd RM
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(45020, 10));
    final MyResourceManager2 rm2 = new MyResourceManager2(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    ((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
    dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
    AMRMTokenSecretManager amrmTokenSecretManagerForRM2 = rm2.getRMContext().getAMRMTokenSecretManager();
    Assert.assertTrue(amrmTokenSecretManagerForRM2.getCurrnetMasterKeyData().getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
    Assert.assertTrue(amrmTokenSecretManagerForRM2.getNextMasterKeyData() == null);
    try {
        UserGroupInformation testUser = UserGroupInformation.createRemoteUser("testUser");
        SecurityUtil.setTokenService(token, rm2.getApplicationMasterService().getBindAddress());
        testUser.addToken(token);
        testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

            @Override
            public ApplicationMasterProtocol run() {
                return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, rm2.getApplicationMasterService().getBindAddress(), conf);
            }
        }).allocate(Records.newRecord(AllocateRequest.class));
        Assert.fail("The old Token should not work");
    } catch (Exception ex) {
        Assert.assertTrue(ex instanceof InvalidToken);
        Assert.assertTrue(ex.getMessage().contains("Invalid AMRMToken from " + token.decodeIdentifier().getApplicationAttemptId()));
    }
    // make sure the recovered AMRMToken works for new RM
    amClient.allocate(0.1f);
    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    amClient.stop();
    rm1.stop();
    rm2.stop();
}