Examples with AbstractYarnScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler used on opensource projects

Example 11 with AbstractYarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler in project hadoop by apache.

the class RMWebServices method getAppActivities.

@GET
@Path("/scheduler/app-activities")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public AppActivitiesInfo getAppActivities(@Context HttpServletRequest hsr, @QueryParam("appId") String appId, @QueryParam("maxTime") String time) {
    YarnScheduler scheduler = rm.getRMContext().getScheduler();
    if (scheduler instanceof AbstractYarnScheduler) {
        AbstractYarnScheduler abstractYarnScheduler = (AbstractYarnScheduler) scheduler;
        ActivitiesManager activitiesManager = abstractYarnScheduler.getActivitiesManager();
        if (null == activitiesManager) {
            String errMessage = "Not Capacity Scheduler";
            return new AppActivitiesInfo(errMessage, appId);
        }
        if (appId == null) {
            String errMessage = "Must provide an application Id";
            return new AppActivitiesInfo(errMessage, null);
        }
        double maxTime = 3.0;
        if (time != null) {
            if (time.contains(".")) {
                maxTime = Double.parseDouble(time);
            } else {
                maxTime = Double.parseDouble(time + ".0");
            }
        }
        ApplicationId applicationId;
        try {
            applicationId = ApplicationId.fromString(appId);
            activitiesManager.turnOnAppActivitiesRecording(applicationId, maxTime);
            AppActivitiesInfo appActivitiesInfo = activitiesManager.getAppActivitiesInfo(applicationId);
            return appActivitiesInfo;
        } catch (Exception e) {
            String errMessage = "Cannot find application with given appId";
            return new AppActivitiesInfo(errMessage, appId);
        }
    }
    return null;
}

Example 12 with AbstractYarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler in project hadoop by apache.

the class TestCapacityScheduler method testMoveAppSameParent.

@Test
public void testMoveAppSameParent() throws Exception {
    MockRM rm = setUpMove();
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm.getResourceScheduler();
    // submit an app
    RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");
    ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()).getCurrentApplicationAttemptId();
    // check preconditions
    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
    assertEquals(1, appsInA1.size());
    String queue = scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue().getQueueName();
    Assert.assertTrue(queue.equals("a1"));
    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    List<ApplicationAttemptId> appsInA2 = scheduler.getAppsInQueue("a2");
    assertTrue(appsInA2.isEmpty());
    // now move the app
    scheduler.moveApplication(app.getApplicationId(), "a2");
    // check postconditions
    appsInA2 = scheduler.getAppsInQueue("a2");
    assertEquals(1, appsInA2.size());
    queue = scheduler.getApplicationAttempt(appsInA2.get(0)).getQueue().getQueueName();
    Assert.assertTrue(queue.equals("a2"));
    appsInA1 = scheduler.getAppsInQueue("a1");
    assertTrue(appsInA1.isEmpty());
    appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    rm.stop();
}

Example 13 with AbstractYarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler in project hadoop by apache.

the class TestCapacityScheduler method testKillAllAppsInQueue.

@Test
public void testKillAllAppsInQueue() throws Exception {
    MockRM rm = setUpMove();
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm.getResourceScheduler();
    // submit an app
    RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");
    ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()).getCurrentApplicationAttemptId();
    // check preconditions
    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
    assertEquals(1, appsInA1.size());
    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    String queue = scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue().getQueueName();
    Assert.assertTrue(queue.equals("a1"));
    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    // now kill the app
    scheduler.killAllAppsInQueue("a1");
    // check postconditions
    rm.waitForState(app.getApplicationId(), RMAppState.KILLED);
    rm.waitForAppRemovedFromScheduler(app.getApplicationId());
    appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.isEmpty());
    appsInA1 = scheduler.getAppsInQueue("a1");
    assertTrue(appsInA1.isEmpty());
    appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.isEmpty());
    rm.stop();
}

Example 14 with AbstractYarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler in project hadoop by apache.

the class TestWorkPreservingRMRestart method testReleasedContainerNotRecovered.

// Test if RM on recovery receives the container release request from AM
// before it receives the container status reported by NM for recovery. this
// container should not be recovered.
@Test(timeout = 50000)
public void testReleasedContainerNotRecovered() throws Exception {
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    rm1 = new MockRM(conf, memStore);
    MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    rm1.start();
    RMApp app1 = rm1.submitApp(1024);
    final MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    // Re-start RM
    conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 8000);
    rm2 = new MockRM(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
    am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
    am1.registerAppAttempt(true);
    // try to release a container before the container is actually recovered.
    final ContainerId runningContainer = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    am1.allocate(null, Arrays.asList(runningContainer));
    // send container statuses to recover the containers
    List<NMContainerStatus> containerStatuses = createNMContainerStatusForApp(am1);
    nm1.registerNode(containerStatuses, null);
    // only the am container should be recovered.
    waitForNumContainersToRecover(1, rm2, am1.getApplicationAttemptId());
    final AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm2.getResourceScheduler();
    // cached release request is cleaned.
    // assertFalse(scheduler.getPendingRelease().contains(runningContainer));
    AllocateResponse response = am1.allocate(null, null);
    // AM gets notified of the completed container.
    boolean receivedCompletedContainer = false;
    for (ContainerStatus status : response.getCompletedContainersStatuses()) {
        if (status.getContainerId().equals(runningContainer)) {
            receivedCompletedContainer = true;
        }
    }
    assertTrue(receivedCompletedContainer);
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            // recovered
            return scheduler.getApplicationAttempt(am1.getApplicationAttemptId()).getPendingRelease().isEmpty() && scheduler.getRMContainer(runningContainer) == null;
        }
    }, 1000, 20000);
}

Example 15 with AbstractYarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler in project hadoop by apache.

the class TestWorkPreservingRMRestart method waitForNumContainersToRecover.

public static void waitForNumContainersToRecover(int num, MockRM rm, ApplicationAttemptId attemptId) throws Exception {
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm.getResourceScheduler();
    SchedulerApplicationAttempt attempt = scheduler.getApplicationAttempt(attemptId);
    while (attempt == null) {
        System.out.println("Wait for scheduler attempt " + attemptId + " to be created");
        Thread.sleep(200);
        attempt = scheduler.getApplicationAttempt(attemptId);
    }
    while (attempt.getLiveContainers().size() < num) {
        System.out.println("Wait for " + num + " containers to recover. currently: " + attempt.getLiveContainers().size());
        Thread.sleep(200);
    }
}