Examples with AppAttemptAddedSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent used on opensource projects

Example 1 with AppAttemptAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent in project hadoop by apache.

the class SLSCapacityScheduler method handle.

@Override
public void handle(SchedulerEvent schedulerEvent) {
    // metrics off
    if (!metricsON) {
        super.handle(schedulerEvent);
        return;
    }
    if (!running)
        running = true;
    // metrics on
    Timer.Context handlerTimer = null;
    Timer.Context operationTimer = null;
    NodeUpdateSchedulerEventWrapper eventWrapper;
    try {
        //if (schedulerEvent instanceof NodeUpdateSchedulerEvent) {
        if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
            eventWrapper = new NodeUpdateSchedulerEventWrapper((NodeUpdateSchedulerEvent) schedulerEvent);
            schedulerEvent = eventWrapper;
            updateQueueWithNodeUpdate(eventWrapper);
        } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
            // check if having AM Container, update resource usage information
            AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent;
            ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID();
            String queue = appQueueMap.get(appAttemptId);
            SchedulerAppReport app = super.getSchedulerAppInfo(appAttemptId);
            if (!app.getLiveContainers().isEmpty()) {
                // have 0 or 1
                // should have one container which is AM container
                RMContainer rmc = app.getLiveContainers().iterator().next();
                updateQueueMetrics(queue, rmc.getContainer().getResource().getMemorySize(), rmc.getContainer().getResource().getVirtualCores());
            }
        }
        handlerTimer = schedulerHandleTimer.time();
        operationTimer = schedulerHandleTimerMap.get(schedulerEvent.getType()).time();
        super.handle(schedulerEvent);
    } finally {
        if (handlerTimer != null)
            handlerTimer.stop();
        if (operationTimer != null)
            operationTimer.stop();
        schedulerHandleCounter.inc();
        schedulerHandleCounterMap.get(schedulerEvent.getType()).inc();
        if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
            SLSRunner.decreaseRemainingApps();
            AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent;
            ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID();
            appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
        } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_ADDED && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {
            AppAttemptAddedSchedulerEvent appAddEvent = (AppAttemptAddedSchedulerEvent) schedulerEvent;
            SchedulerApplication app = applications.get(appAddEvent.getApplicationAttemptId().getApplicationId());
            appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue().getQueueName());
        }
    }
}

Example 2 with AppAttemptAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent in project hadoop by apache.

the class TestAppRunnability method testAppAdditionAndRemoval.

@Test
public void testAppAdditionAndRemoval() throws Exception {
    ApplicationAttemptId attemptId = createAppAttemptId(1, 1);
    AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default", "user1");
    scheduler.handle(appAddedEvent);
    AppAttemptAddedSchedulerEvent attemptAddedEvent = new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
    scheduler.handle(attemptAddedEvent);
    // Scheduler should have two queues (the default and the one created for
    // user1)
    assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
    // That queue should have one app
    assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true).getNumRunnableApps());
    AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);
    // Now remove app
    scheduler.handle(appRemovedEvent1);
    // Queue should have no apps
    assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true).getNumRunnableApps());
}

Example 3 with AppAttemptAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testMoveAfterRemoval.

@Test(expected = YarnException.class)
public void testMoveAfterRemoval() throws Exception {
    // convenience var
    String testUser = "user1";
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    ApplicationAttemptId attemptId = createAppAttemptId(1, 1);
    AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), testUser, testUser);
    scheduler.handle(appAddedEvent);
    AppAttemptAddedSchedulerEvent attemptAddedEvent = new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
    scheduler.handle(attemptAddedEvent);
    // Get a handle on the attempt.
    FSAppAttempt attempt = scheduler.getSchedulerApp(attemptId);
    AppAttemptRemovedSchedulerEvent attemptRemovedEvent = new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);
    // Remove the app attempt
    scheduler.handle(attemptRemovedEvent);
    // Make sure the app attempt is not in the queue and stopped.
    List<ApplicationAttemptId> attemptList = scheduler.getAppsInQueue(testUser);
    assertNotNull("Queue missing", attemptList);
    assertFalse("Attempt should not be in the queue", attemptList.contains(attemptId));
    assertTrue("Attempt should have been stopped", attempt.isStopped());
    // The attempt should still show the original queue info.
    assertTrue("Attempt queue has changed", attempt.getQueue().getName().endsWith(testUser));
    // Now move the app: not using an event since there is none
    // in the scheduler. This should throw.
    scheduler.moveApplication(attemptId.getApplicationId(), "default");
}

Example 4 with AppAttemptAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testDoubleRemoval.

@Test
public void testDoubleRemoval() throws Exception {
    // convenience var
    String testUser = "user1";
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    ApplicationAttemptId attemptId = createAppAttemptId(1, 1);
    // The placement rule will add the app to the user based queue but the
    // passed in queue must exist.
    AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), testUser, testUser);
    scheduler.handle(appAddedEvent);
    AppAttemptAddedSchedulerEvent attemptAddedEvent = new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
    scheduler.handle(attemptAddedEvent);
    // Get a handle on the attempt.
    FSAppAttempt attempt = scheduler.getSchedulerApp(attemptId);
    AppAttemptRemovedSchedulerEvent attemptRemovedEvent = new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);
    // Make sure the app attempt is in the queue.
    List<ApplicationAttemptId> attemptList = scheduler.getAppsInQueue(testUser);
    assertNotNull("Queue missing", attemptList);
    assertTrue("Attempt should be in the queue", attemptList.contains(attemptId));
    assertFalse("Attempt is stopped", attempt.isStopped());
    // Now remove the app attempt
    scheduler.handle(attemptRemovedEvent);
    // The attempt is not in the queue, and stopped
    attemptList = scheduler.getAppsInQueue(testUser);
    assertFalse("Attempt should not be in the queue", attemptList.contains(attemptId));
    assertTrue("Attempt should have been stopped", attempt.isStopped());
    // Now remove the app attempt again, since it is stopped nothing happens.
    scheduler.handle(attemptRemovedEvent);
    // The attempt should still show the original queue info.
    assertTrue("Attempt queue has changed", attempt.getQueue().getName().endsWith(testUser));
}

Example 5 with AppAttemptAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent in project hadoop by apache.

the class TestLeafQueue method testAppAttemptMetrics.

@Test
public void testAppAttemptMetrics() throws Exception {
    // Manipulate queue 'a'
    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));
    // Users
    final String user_0 = "user_0";
    // Submit applications
    final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 1);
    AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(), a.getQueueName(), user_0);
    cs.handle(addAppEvent);
    AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
    cs.handle(addAttemptEvent);
    AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FAILED, false);
    cs.handle(event);
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    // Attempt the same application again
    final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(0, 2);
    FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, null, spyRMContext);
    app_1.setAMResource(Resource.newInstance(100, 1));
    // same user
    a.submitApplicationAttempt(app_1, user_0);
    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(1, a.getMetrics().getAppsPending());
    assertEquals(1, a.getUser(user_0).getActiveApplications());
    assertEquals(app_1.getAMResource().getMemorySize(), a.getMetrics().getUsedAMResourceMB());
    assertEquals(app_1.getAMResource().getVirtualCores(), a.getMetrics().getUsedAMResourceVCores());
    event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FINISHED, false);
    cs.handle(event);
    AppRemovedSchedulerEvent rEvent = new AppRemovedSchedulerEvent(appAttemptId_0.getApplicationId(), RMAppState.FINISHED);
    cs.handle(rEvent);
    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    assertEquals(1, a.getMetrics().getAppsCompleted());
    QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
    assertEquals(1, userMetrics.getAppsSubmitted());
}