Examples with RMAppAttempt org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt used on opensource projects

Example 71 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestFairScheduler method testRefreshQueuesWhenRMHA.

@Test(timeout = 120000)
public void testRefreshQueuesWhenRMHA() throws Exception {
    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
    conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
    conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false);
    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
    HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource.REQUEST_BY_USER);
    // 1. start a standby RM, file 'ALLOC_FILE' is empty, so there is no queues
    MockRM rm1 = new MockRM(conf, null);
    rm1.init(conf);
    rm1.start();
    rm1.getAdminService().transitionToStandby(requestInfo);
    // 2. add a new queue "test_queue"
    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"test_queue\">");
    out.println("  <maxRunningApps>3</maxRunningApps>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // 3. start a active RM
    MockRM rm2 = new MockRM(conf, memStore);
    rm2.init(conf);
    rm2.start();
    MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());
    nm.registerNode();
    rm2.getAdminService().transitionToActive(requestInfo);
    // 4. submit a app to the new added queue "test_queue"
    RMApp app = rm2.submitApp(200, "test_app", "user", null, "test_queue");
    RMAppAttempt attempt0 = app.getCurrentAppAttempt();
    nm.nodeHeartbeat(true);
    MockAM am0 = rm2.sendAMLaunched(attempt0.getAppAttemptId());
    am0.registerAppAttempt();
    assertEquals("root.test_queue", app.getQueue());
    // 5. Transit rm1 to active, recover app
    ((RMContextImpl) rm1.getRMContext()).setStateStore(memStore);
    rm1.getAdminService().transitionToActive(requestInfo);
    rm1.drainEvents();
    assertEquals(1, rm1.getRMContext().getRMApps().size());
    RMApp recoveredApp = rm1.getRMContext().getRMApps().values().iterator().next();
    assertEquals("root.test_queue", recoveredApp.getQueue());
    rm1.stop();
    rm2.stop();
}

Example 72 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class FairSchedulerTestBase method createSchedulingRequest.

protected ApplicationAttemptId createSchedulingRequest(int memory, int vcores, String queueId, String userId, int numContainers, int priority) {
    ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
    scheduler.addApplication(id.getApplicationId(), queueId, userId, false);
    // and no app is added.
    if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) {
        scheduler.addApplicationAttempt(id, false, false);
    }
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, priority, numContainers, true);
    ask.add(request);
    RMApp rmApp = mock(RMApp.class);
    RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
    when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
    when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn(new RMAppAttemptMetrics(id, resourceManager.getRMContext()));
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(submissionContext.getUnmanagedAM()).thenReturn(false);
    when(rmAppAttempt.getSubmissionContext()).thenReturn(submissionContext);
    Container container = mock(Container.class);
    when(rmAppAttempt.getMasterContainer()).thenReturn(container);
    resourceManager.getRMContext().getRMApps().put(id.getApplicationId(), rmApp);
    scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    scheduler.update();
    return id;
}

Example 73 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class FairSchedulerTestBase method createSchedulingRequest.

protected ApplicationAttemptId createSchedulingRequest(String queueId, String userId, List<ResourceRequest> ask) {
    ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
    scheduler.addApplication(id.getApplicationId(), queueId, userId, false);
    // and no app is added.
    if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) {
        scheduler.addApplicationAttempt(id, false, false);
    }
    RMApp rmApp = mock(RMApp.class);
    RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
    when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
    when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn(new RMAppAttemptMetrics(id, resourceManager.getRMContext()));
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(submissionContext.getUnmanagedAM()).thenReturn(false);
    when(rmAppAttempt.getSubmissionContext()).thenReturn(submissionContext);
    resourceManager.getRMContext().getRMApps().put(id.getApplicationId(), rmApp);
    scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    return id;
}

Example 74 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class MockRMWithCustomAMLauncher method createAMLauncher.

@Override
protected ApplicationMasterLauncher createAMLauncher() {
    return new ApplicationMasterLauncher(getRMContext()) {

        @Override
        protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) {
            return new AMLauncher(context, application, event, getConfig()) {

                @Override
                protected ContainerManagementProtocol getContainerMgrProxy(ContainerId containerId) {
                    return containerManager;
                }

                @Override
                protected Token<AMRMTokenIdentifier> createAndSetAMRMToken() {
                    Token<AMRMTokenIdentifier> amRmToken = super.createAndSetAMRMToken();
                    InetSocketAddress serviceAddr = getConfig().getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
                    SecurityUtil.setTokenService(amRmToken, serviceAddr);
                    return amRmToken;
                }
            };
        }
    };
}

Example 75 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestApplicationCleanup method testAppCleanup.

@SuppressWarnings("resource")
@Test
public void testAppCleanup() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    MockRM rm = new MockRM();
    rm.start();
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000);
    RMApp app = rm.submitApp(2000);
    //kick the scheduling
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
    am.registerAppAttempt();
    //request for containers
    int request = 2;
    am.allocate("127.0.0.1", 1000, request, new ArrayList<ContainerId>());
    //kick the scheduler
    nm1.nodeHeartbeat(true);
    List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    int contReceived = conts.size();
    int waitCount = 0;
    while (contReceived < request && waitCount++ < 200) {
        LOG.info("Got " + contReceived + " containers. Waiting to get " + request);
        Thread.sleep(100);
        conts = am.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        contReceived += conts.size();
        nm1.nodeHeartbeat(true);
    }
    Assert.assertEquals(request, contReceived);
    am.unregisterAppAttempt();
    NodeHeartbeatResponse resp = nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    //currently only containers are cleaned via this
    //AM container is cleaned via container launcher
    resp = nm1.nodeHeartbeat(true);
    List<ContainerId> containersToCleanup = resp.getContainersToCleanup();
    List<ApplicationId> appsToCleanup = resp.getApplicationsToCleanup();
    int numCleanedContainers = containersToCleanup.size();
    int numCleanedApps = appsToCleanup.size();
    waitCount = 0;
    while ((numCleanedContainers < 2 || numCleanedApps < 1) && waitCount++ < 200) {
        LOG.info("Waiting to get cleanup events.. cleanedConts: " + numCleanedContainers + " cleanedApps: " + numCleanedApps);
        Thread.sleep(100);
        resp = nm1.nodeHeartbeat(true);
        List<ContainerId> deltaContainersToCleanup = resp.getContainersToCleanup();
        List<ApplicationId> deltaAppsToCleanup = resp.getApplicationsToCleanup();
        // Add the deltas to the global list
        containersToCleanup.addAll(deltaContainersToCleanup);
        appsToCleanup.addAll(deltaAppsToCleanup);
        // Update counts now
        numCleanedContainers = containersToCleanup.size();
        numCleanedApps = appsToCleanup.size();
    }
    Assert.assertEquals(1, appsToCleanup.size());
    Assert.assertEquals(app.getApplicationId(), appsToCleanup.get(0));
    Assert.assertEquals(1, numCleanedApps);
    Assert.assertEquals(2, numCleanedContainers);
    rm.stop();
}