Search in sources :

Example 51 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestRemoteAppChecker method testRunningApp.

@Test
public void testRunningApp() throws Exception {
    YarnClient client = createCheckerWithMockedClient();
    ApplicationId id = ApplicationId.newInstance(1, 1);
    // create a report and set the state to an active one
    ApplicationReport report = new ApplicationReportPBImpl();
    report.setYarnApplicationState(YarnApplicationState.ACCEPTED);
    doReturn(report).when(client).getApplicationReport(id);
    assertTrue(checker.isApplicationActive(id));
}
Also used : ApplicationReport(org.apache.hadoop.yarn.api.records.ApplicationReport) ApplicationReportPBImpl(org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) YarnClient(org.apache.hadoop.yarn.client.api.YarnClient) Test(org.junit.Test)

Example 52 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestRemoteAppChecker method createCheckerWithMockedClient.

/**
   * Creates/initializes/starts a RemoteAppChecker with a spied
   * DummyYarnClientImpl.
   * 
   * @return the spied DummyYarnClientImpl in the created AppChecker
   */
private YarnClient createCheckerWithMockedClient() {
    YarnClient client = spy(new DummyYarnClientImpl());
    checker = new RemoteAppChecker(client);
    checker.init(new Configuration());
    checker.start();
    return client;
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) YarnClient(org.apache.hadoop.yarn.client.api.YarnClient)

Example 53 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyE2E.

/*
   * This test validates register, allocate and finish of an application through
   * the AMRMPRoxy.
   */
@Test(timeout = 120000)
public void testAMRMProxyE2E() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testAMRMProxyE2E", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        // the client has to connect to AMRMProxy
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit application
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        LOG.info("testAMRMProxyE2E - Register Application Master");
        RegisterApplicationMasterResponse responseRegister = client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        Assert.assertNotNull(responseRegister);
        Assert.assertNotNull(responseRegister.getQueue());
        Assert.assertNotNull(responseRegister.getApplicationACLs());
        Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
        Assert.assertNotNull(responseRegister.getContainersFromPreviousAttempts());
        Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
        Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
        RMApp rmApp = cluster.getResourceManager().getRMContext().getRMApps().get(appId);
        Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
        LOG.info("testAMRMProxyE2E - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        AllocateResponse allocResponse = client.allocate(request);
        Assert.assertNotNull(allocResponse);
        Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
        request.setAskList(new ArrayList<ResourceRequest>());
        request.setResponseId(request.getResponseId() + 1);
        Thread.sleep(1000);
        // RM should allocate container within 2 calls to allocate()
        allocResponse = client.allocate(request);
        Assert.assertNotNull(allocResponse);
        Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
        LOG.info("testAMRMPRoxy - Finish Application Master");
        FinishApplicationMasterResponse responseFinish = client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
        Assert.assertNotNull(responseFinish);
        Thread.sleep(500);
        Assert.assertNotEquals(RMAppState.FINISHED, rmApp.getState());
    }
}
Also used : RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) ApplicationMasterProtocol(org.apache.hadoop.yarn.api.ApplicationMasterProtocol) FinishApplicationMasterResponse(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) YarnClient(org.apache.hadoop.yarn.client.api.YarnClient) AllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) RegisterApplicationMasterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse) ResourceRequest(org.apache.hadoop.yarn.api.records.ResourceRequest) MiniYARNCluster(org.apache.hadoop.yarn.server.MiniYARNCluster) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Test(org.junit.Test)

Example 54 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyTokenRenewal.

/*
   * This test validates the token renewal from the AMRMPRoxy. The test verifies
   * that the received token from AMRMProxy is different from the previous one
   * within 5 requests.
   */
@Test(timeout = 120000)
public void testAMRMProxyTokenRenewal() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 1500);
        // RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS should be at least
        // RM_AM_EXPIRY_INTERVAL_MS * 1.5 *3
        conf.setInt(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 6);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        LOG.info("testAMRMProxyTokenRenewal - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        Token lastToken = null;
        AllocateResponse response = null;
        for (int i = 0; i < 5; i++) {
            response = client.allocate(request);
            request.setResponseId(request.getResponseId() + 1);
            if (response.getAMRMToken() != null && !response.getAMRMToken().equals(lastToken)) {
                break;
            }
            lastToken = response.getAMRMToken();
            // Time slot to be sure the AMRMProxy renew the token
            Thread.sleep(1500);
        }
        Assert.assertFalse(response.getAMRMToken().equals(lastToken));
        LOG.info("testAMRMPRoxy - Finish Application Master");
        client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    }
}
Also used : AllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) ApplicationMasterProtocol(org.apache.hadoop.yarn.api.ApplicationMasterProtocol) Token(org.apache.hadoop.yarn.api.records.Token) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) MiniYARNCluster(org.apache.hadoop.yarn.server.MiniYARNCluster) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) YarnClient(org.apache.hadoop.yarn.client.api.YarnClient) Test(org.junit.Test)

Example 55 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestAMRMProxy method testE2ETokenSwap.

/*
   * This test validates that an AM cannot register directly to the RM, with the
   * token provided by the AMRMProxy.
   */
@Test(timeout = 120000)
public void testE2ETokenSwap() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenSwap", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        cluster.init(conf);
        cluster.start();
        // the client will connect to the RM with the token provided by AMRMProxy
        final Configuration yarnConf = cluster.getConfig();
        rmClient.init(yarnConf);
        rmClient.start();
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        try {
            client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
            Assert.fail();
        } catch (IOException e) {
            Assert.assertTrue(e.getMessage().startsWith("Invalid AMRMToken from appattempt_"));
        }
    }
}
Also used : YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ApplicationMasterProtocol(org.apache.hadoop.yarn.api.ApplicationMasterProtocol) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) IOException(java.io.IOException) MiniYARNCluster(org.apache.hadoop.yarn.server.MiniYARNCluster) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) YarnClient(org.apache.hadoop.yarn.client.api.YarnClient) Test(org.junit.Test)

Aggregations

YarnClient (org.apache.hadoop.yarn.client.api.YarnClient)89 Test (org.junit.Test)51 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)50 Configuration (org.apache.hadoop.conf.Configuration)45 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)37 ApplicationReport (org.apache.hadoop.yarn.api.records.ApplicationReport)21 CapacitySchedulerConfiguration (org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration)18 IOException (java.io.IOException)17 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)15 MiniYARNCluster (org.apache.hadoop.yarn.server.MiniYARNCluster)15 YarnException (org.apache.hadoop.yarn.exceptions.YarnException)14 Path (org.apache.hadoop.fs.Path)13 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)13 FileSystem (org.apache.hadoop.fs.FileSystem)11 Matchers.anyString (org.mockito.Matchers.anyString)11 UserGroupInformation (org.apache.hadoop.security.UserGroupInformation)9 NodeId (org.apache.hadoop.yarn.api.records.NodeId)9 ArrayList (java.util.ArrayList)8 LocalFileSystem (org.apache.hadoop.fs.LocalFileSystem)8 ReservationSubmissionRequest (org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest)7