Examples with AppAddedSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent used on opensource projects

Example 16 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class FairSchedulerTestBase method createApplicationWithAMResource.

protected void createApplicationWithAMResource(ApplicationAttemptId attId, String queue, String user, Resource amResource) {
    RMContext rmContext = resourceManager.getRMContext();
    ApplicationId appId = attId.getApplicationId();
    RMApp rmApp = new RMAppImpl(appId, rmContext, conf, null, user, null, ApplicationSubmissionContext.newInstance(appId, null, queue, null, mock(ContainerLaunchContext.class), false, false, 0, amResource, null), scheduler, null, 0, null, null, null);
    rmContext.getRMApps().put(appId, rmApp);
    RMAppEvent event = new RMAppEvent(appId, RMAppEventType.START);
    resourceManager.getRMContext().getRMApps().get(appId).handle(event);
    event = new RMAppEvent(appId, RMAppEventType.APP_NEW_SAVED);
    resourceManager.getRMContext().getRMApps().get(appId).handle(event);
    event = new RMAppEvent(appId, RMAppEventType.APP_ACCEPTED);
    resourceManager.getRMContext().getRMApps().get(appId).handle(event);
    AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appId, queue, user);
    scheduler.handle(appAddedEvent);
    AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent(attId, false);
    scheduler.handle(attempAddedEvent);
}

Example 17 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class TestSchedulerPlanFollowerBase method testPlanFollower.

protected void testPlanFollower(boolean isMove) throws PlanningException, InterruptedException, AccessControlException {
    // Initialize plan based on move flag
    plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent, scheduler.getClusterResource(), 1L, res, scheduler.getMinimumResourceCapability(), maxAlloc, "dedicated", null, isMove, context);
    // add a few reservations to the plan
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int[] f1 = { 10, 10, 10, 10, 10 };
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(0, 0 + f1.length + 1, f1.length);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, rDef, "u3", "dedicated", 0, 0 + f1.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, f1), res, minAlloc), false));
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, rDef, "u3", "dedicated", 3, 3 + f1.length, ReservationSystemTestUtil.generateAllocation(3L, 1L, f1), res, minAlloc), false));
    ReservationId r3 = ReservationId.newInstance(ts, 3);
    int[] f2 = { 0, 10, 20, 10, 0 };
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, rDef, "u4", "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil.generateAllocation(10L, 1L, f2), res, minAlloc), false));
    AbstractSchedulerPlanFollower planFollower = createPlanFollower();
    when(mClock.getTime()).thenReturn(0L);
    planFollower.run();
    Queue q = getReservationQueue(r1.toString());
    assertReservationQueueExists(r1);
    // submit an app to r1
    String user_0 = "test-user";
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId_0 = ApplicationAttemptId.newInstance(appId, 0);
    AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, q.getQueueName(), user_0);
    scheduler.handle(addAppEvent);
    AppAttemptAddedSchedulerEvent appAttemptAddedEvent = new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
    scheduler.handle(appAttemptAddedEvent);
    // initial default reservation queue should have no apps
    Queue defQ = getDefaultQueue();
    Assert.assertEquals(0, getNumberOfApplications(defQ));
    assertReservationQueueExists(r1, 0.1, 0.1);
    Assert.assertEquals(1, getNumberOfApplications(q));
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueDoesNotExist(r3);
    when(mClock.getTime()).thenReturn(3L);
    planFollower.run();
    Assert.assertEquals(0, getNumberOfApplications(defQ));
    assertReservationQueueExists(r1, 0.1, 0.1);
    Assert.assertEquals(1, getNumberOfApplications(q));
    assertReservationQueueExists(r2, 0.1, 0.1);
    assertReservationQueueDoesNotExist(r3);
    when(mClock.getTime()).thenReturn(10L);
    planFollower.run();
    q = getReservationQueue(r1.toString());
    if (isMove) {
        // app should have been moved to default reservation queue
        Assert.assertEquals(1, getNumberOfApplications(defQ));
        assertNull(q);
    } else {
        // app should be killed
        Assert.assertEquals(0, getNumberOfApplications(defQ));
        assertNotNull(q);
        AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.KILLED, false);
        scheduler.handle(appAttemptRemovedEvent);
    }
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueExists(r3, 0, 1.0);
    when(mClock.getTime()).thenReturn(11L);
    planFollower.run();
    if (isMove) {
        // app should have been moved to default reservation queue
        Assert.assertEquals(1, getNumberOfApplications(defQ));
    } else {
        // app should be killed
        Assert.assertEquals(0, getNumberOfApplications(defQ));
    }
    assertReservationQueueDoesNotExist(r1);
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueExists(r3, 0.1, 0.1);
    when(mClock.getTime()).thenReturn(12L);
    planFollower.run();
    assertReservationQueueDoesNotExist(r1);
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueExists(r3, 0.2, 0.2);
    when(mClock.getTime()).thenReturn(16L);
    planFollower.run();
    assertReservationQueueDoesNotExist(r1);
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueDoesNotExist(r3);
    verifyCapacity(defQ);
}

Example 18 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class Application method submit.

@SuppressWarnings("deprecation")
public synchronized void submit() throws IOException, YarnException {
    ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    context.setApplicationId(this.applicationId);
    context.setQueue(this.queue);
    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    context.setAMContainerSpec(amContainer);
    context.setResource(Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
    SubmitApplicationRequest request = recordFactory.newRecordInstance(SubmitApplicationRequest.class);
    request.setApplicationSubmissionContext(context);
    final ResourceScheduler scheduler = resourceManager.getResourceScheduler();
    resourceManager.getClientRMService().submitApplication(request);
    RMAppEvent event = new RMAppEvent(this.applicationId, RMAppEventType.START);
    resourceManager.getRMContext().getRMApps().get(applicationId).handle(event);
    event = new RMAppEvent(this.applicationId, RMAppEventType.APP_NEW_SAVED);
    resourceManager.getRMContext().getRMApps().get(applicationId).handle(event);
    event = new RMAppEvent(this.applicationId, RMAppEventType.APP_ACCEPTED);
    resourceManager.getRMContext().getRMApps().get(applicationId).handle(event);
    // Notify scheduler
    AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId, this.queue, "user");
    scheduler.handle(addAppEvent);
    AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false);
    scheduler.handle(addAttemptEvent);
}

Example 19 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class TestCapacityScheduler method testRemoveAttemptMoveAdded.

@Test
public void testRemoveAttemptMoveAdded() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, CapacityScheduler.class);
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
    // Create Mock RM
    MockRM rm = new MockRM(getCapacityConfiguration(conf));
    CapacityScheduler sch = (CapacityScheduler) rm.getResourceScheduler();
    // add node
    Resource newResource = Resource.newInstance(4 * GB, 1);
    RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1");
    SchedulerEvent addNode = new NodeAddedSchedulerEvent(node);
    sch.handle(addNode);
    // create appid
    ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    RMAppAttemptMetrics attemptMetric = new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
    RMAppImpl app = mock(RMAppImpl.class);
    when(app.getApplicationId()).thenReturn(appId);
    RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
    Container container = mock(Container.class);
    when(attempt.getMasterContainer()).thenReturn(container);
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(attempt.getSubmissionContext()).thenReturn(submissionContext);
    when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
    when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
    when(app.getCurrentAppAttempt()).thenReturn(attempt);
    rm.getRMContext().getRMApps().put(appId, app);
    // Add application
    SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, "a1", "user");
    sch.handle(addAppEvent);
    // Add application attempt
    SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    sch.handle(addAttemptEvent);
    // get Queues
    CSQueue queueA1 = sch.getQueue("a1");
    CSQueue queueB = sch.getQueue("b");
    CSQueue queueB1 = sch.getQueue("b1");
    // add Running rm container and simulate live containers to a1
    ContainerId newContainerId = ContainerId.newContainerId(appAttemptId, 2);
    RMContainerImpl rmContainer = mock(RMContainerImpl.class);
    when(rmContainer.getState()).thenReturn(RMContainerState.RUNNING);
    Container container2 = mock(Container.class);
    when(rmContainer.getContainer()).thenReturn(container2);
    Resource resource = Resource.newInstance(1024, 1);
    when(container2.getResource()).thenReturn(resource);
    when(rmContainer.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
    when(container2.getNodeId()).thenReturn(node.getNodeID());
    when(container2.getId()).thenReturn(newContainerId);
    when(rmContainer.getNodeLabelExpression()).thenReturn(RMNodeLabelsManager.NO_LABEL);
    when(rmContainer.getContainerId()).thenReturn(newContainerId);
    sch.getApplicationAttempt(appAttemptId).getLiveContainersMap().put(newContainerId, rmContainer);
    QueueMetrics queueA1M = queueA1.getMetrics();
    queueA1M.incrPendingResources("user1", 1, resource);
    queueA1M.allocateResources("user1", resource);
    // remove attempt
    sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId, RMAppAttemptState.KILLED, true));
    // Move application to queue b1
    sch.moveApplication(appId, "b1");
    // Check queue metrics after move
    Assert.assertEquals(0, queueA1.getNumApplications());
    Assert.assertEquals(1, queueB.getNumApplications());
    Assert.assertEquals(0, queueB1.getNumApplications());
    // Release attempt add event
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(appId, 2);
    SchedulerEvent addAttemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, true);
    sch.handle(addAttemptEvent2);
    // Check metrics after attempt added
    Assert.assertEquals(0, queueA1.getNumApplications());
    Assert.assertEquals(1, queueB.getNumApplications());
    Assert.assertEquals(1, queueB1.getNumApplications());
    QueueMetrics queueB1M = queueB1.getMetrics();
    QueueMetrics queueBM = queueB.getMetrics();
    // Verify allocation MB of current state
    Assert.assertEquals(0, queueA1M.getAllocatedMB());
    Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores());
    Assert.assertEquals(1024, queueB1M.getAllocatedMB());
    Assert.assertEquals(1, queueB1M.getAllocatedVirtualCores());
    // remove attempt
    sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId2, RMAppAttemptState.FINISHED, false));
    Assert.assertEquals(0, queueA1M.getAllocatedMB());
    Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores());
    Assert.assertEquals(0, queueB1M.getAllocatedMB());
    Assert.assertEquals(0, queueB1M.getAllocatedVirtualCores());
    verifyQueueMetrics(queueB1M);
    verifyQueueMetrics(queueBM);
    // Verify queue A1 metrics
    verifyQueueMetrics(queueA1M);
    rm.close();
}

Example 20 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class TestApplicationPriority method testApplicationOrderingWithPriority.

@Test
public void testApplicationOrderingWithPriority() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    LeafQueue q = (LeafQueue) cs.getQueue("default");
    Assert.assertNotNull(q);
    String host = "127.0.0.1";
    RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(16 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));
    // add app 1 start
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1);
    RMAppAttemptMetrics attemptMetric1 = new RMAppAttemptMetrics(appAttemptId1, rm.getRMContext());
    RMAppImpl app1 = mock(RMAppImpl.class);
    when(app1.getApplicationId()).thenReturn(appId1);
    RMAppAttemptImpl attempt1 = mock(RMAppAttemptImpl.class);
    when(attempt1.getAppAttemptId()).thenReturn(appAttemptId1);
    when(attempt1.getRMAppAttemptMetrics()).thenReturn(attemptMetric1);
    when(app1.getCurrentAppAttempt()).thenReturn(attempt1);
    rm.getRMContext().getRMApps().put(appId1, app1);
    SchedulerEvent addAppEvent1 = new AppAddedSchedulerEvent(appId1, "default", "user", null, Priority.newInstance(5));
    cs.handle(addAppEvent1);
    SchedulerEvent addAttemptEvent1 = new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    cs.handle(addAttemptEvent1);
    // add app1 end
    // add app2 begin
    ApplicationId appId2 = BuilderUtils.newApplicationId(100, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(appId2, 1);
    RMAppAttemptMetrics attemptMetric2 = new RMAppAttemptMetrics(appAttemptId2, rm.getRMContext());
    RMAppImpl app2 = mock(RMAppImpl.class);
    when(app2.getApplicationId()).thenReturn(appId2);
    RMAppAttemptImpl attempt2 = mock(RMAppAttemptImpl.class);
    when(attempt2.getAppAttemptId()).thenReturn(appAttemptId2);
    when(attempt2.getRMAppAttemptMetrics()).thenReturn(attemptMetric2);
    when(app2.getCurrentAppAttempt()).thenReturn(attempt2);
    rm.getRMContext().getRMApps().put(appId2, app2);
    SchedulerEvent addAppEvent2 = new AppAddedSchedulerEvent(appId2, "default", "user", null, Priority.newInstance(8));
    cs.handle(addAppEvent2);
    SchedulerEvent addAttemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
    cs.handle(addAttemptEvent2);
    // add app end
    // Now, the first assignment will be for app2 since app2 is of highest
    // priority
    assertEquals(q.getApplications().size(), 2);
    assertEquals(q.getApplications().iterator().next().getApplicationAttemptId(), appAttemptId2);
    rm.stop();
}