Examples with ApplicationMasterProtocol org.apache.hadoop.yarn.api.ApplicationMasterProtocol used on opensource projects

Example 16 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestLocalContainerAllocator method testRMConnectionRetry.

@Test
public void testRMConnectionRetry() throws Exception {
    // verify the connection exception is thrown
    // if we haven't exhausted the retry interval
    ApplicationMasterProtocol mockScheduler = mock(ApplicationMasterProtocol.class);
    when(mockScheduler.allocate(isA(AllocateRequest.class))).thenThrow(RPCUtil.getRemoteException(new IOException("forcefail")));
    Configuration conf = new Configuration();
    LocalContainerAllocator lca = new StubbedLocalContainerAllocator(mockScheduler);
    lca.init(conf);
    lca.start();
    try {
        lca.heartbeat();
        Assert.fail("heartbeat was supposed to throw");
    } catch (YarnException e) {
    // YarnException is expected
    } finally {
        lca.stop();
    }
    // verify YarnRuntimeException is thrown when the retry interval has expired
    conf.setLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, 0);
    lca = new StubbedLocalContainerAllocator(mockScheduler);
    lca.init(conf);
    lca.start();
    try {
        lca.heartbeat();
        Assert.fail("heartbeat was supposed to throw");
    } catch (YarnRuntimeException e) {
    // YarnRuntimeException is expected
    } finally {
        lca.stop();
    }
}

Example 17 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyE2E.

/*
   * This test validates register, allocate and finish of an application through
   * the AMRMPRoxy.
   */
@Test(timeout = 120000)
public void testAMRMProxyE2E() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testAMRMProxyE2E", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        // the client has to connect to AMRMProxy
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit application
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        LOG.info("testAMRMProxyE2E - Register Application Master");
        RegisterApplicationMasterResponse responseRegister = client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        Assert.assertNotNull(responseRegister);
        Assert.assertNotNull(responseRegister.getQueue());
        Assert.assertNotNull(responseRegister.getApplicationACLs());
        Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
        Assert.assertNotNull(responseRegister.getContainersFromPreviousAttempts());
        Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
        Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
        RMApp rmApp = cluster.getResourceManager().getRMContext().getRMApps().get(appId);
        Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
        LOG.info("testAMRMProxyE2E - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        AllocateResponse allocResponse = client.allocate(request);
        Assert.assertNotNull(allocResponse);
        Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
        request.setAskList(new ArrayList<ResourceRequest>());
        request.setResponseId(request.getResponseId() + 1);
        Thread.sleep(1000);
        // RM should allocate container within 2 calls to allocate()
        allocResponse = client.allocate(request);
        Assert.assertNotNull(allocResponse);
        Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
        LOG.info("testAMRMPRoxy - Finish Application Master");
        FinishApplicationMasterResponse responseFinish = client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
        Assert.assertNotNull(responseFinish);
        Thread.sleep(500);
        Assert.assertNotEquals(RMAppState.FINISHED, rmApp.getState());
    }
}

Example 18 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyTokenRenewal.

/*
   * This test validates the token renewal from the AMRMPRoxy. The test verifies
   * that the received token from AMRMProxy is different from the previous one
   * within 5 requests.
   */
@Test(timeout = 120000)
public void testAMRMProxyTokenRenewal() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 1500);
        // RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS should be at least
        // RM_AM_EXPIRY_INTERVAL_MS * 1.5 *3
        conf.setInt(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 6);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        LOG.info("testAMRMProxyTokenRenewal - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        Token lastToken = null;
        AllocateResponse response = null;
        for (int i = 0; i < 5; i++) {
            response = client.allocate(request);
            request.setResponseId(request.getResponseId() + 1);
            if (response.getAMRMToken() != null && !response.getAMRMToken().equals(lastToken)) {
                break;
            }
            lastToken = response.getAMRMToken();
            // Time slot to be sure the AMRMProxy renew the token
            Thread.sleep(1500);
        }
        Assert.assertFalse(response.getAMRMToken().equals(lastToken));
        LOG.info("testAMRMPRoxy - Finish Application Master");
        client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    }
}

Example 19 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestAMRMProxy method testE2ETokenSwap.

/*
   * This test validates that an AM cannot register directly to the RM, with the
   * token provided by the AMRMProxy.
   */
@Test(timeout = 120000)
public void testE2ETokenSwap() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenSwap", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        cluster.init(conf);
        cluster.start();
        // the client will connect to the RM with the token provided by AMRMProxy
        final Configuration yarnConf = cluster.getConfig();
        rmClient.init(yarnConf);
        rmClient.start();
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        try {
            client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
            Assert.fail();
        } catch (IOException e) {
            Assert.assertTrue(e.getMessage().startsWith("Invalid AMRMToken from appattempt_"));
        }
    }
}

Example 20 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestRPCFactories method testPbClientFactory.

private void testPbClientFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    System.err.println(addr.getHostName() + addr.getPort());
    Configuration conf = new Configuration();
    ApplicationMasterProtocol instance = new AMRMProtocolTestImpl();
    Server server = null;
    try {
        server = RpcServerFactoryPBImpl.get().getServer(ApplicationMasterProtocol.class, instance, addr, conf, null, 1);
        server.start();
        System.err.println(server.getListenerAddress());
        System.err.println(NetUtils.getConnectAddress(server));
        ApplicationMasterProtocol amrmClient = null;
        try {
            amrmClient = (ApplicationMasterProtocol) RpcClientFactoryPBImpl.get().getClient(ApplicationMasterProtocol.class, 1, NetUtils.getConnectAddress(server), conf);
        } catch (YarnRuntimeException e) {
            e.printStackTrace();
            Assert.fail("Failed to create client");
        }
    } catch (YarnRuntimeException e) {
        e.printStackTrace();
        Assert.fail("Failed to create server");
    } finally {
        if (server != null) {
            server.stop();
        }
    }
}