Examples with AppAddedSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent used on opensource projects

Example 1 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testMoveAfterRemoval.

@Test(expected = YarnException.class)
public void testMoveAfterRemoval() throws Exception {
    // convenience var
    String testUser = "user1";
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    ApplicationAttemptId attemptId = createAppAttemptId(1, 1);
    AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), testUser, testUser);
    scheduler.handle(appAddedEvent);
    AppAttemptAddedSchedulerEvent attemptAddedEvent = new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
    scheduler.handle(attemptAddedEvent);
    // Get a handle on the attempt.
    FSAppAttempt attempt = scheduler.getSchedulerApp(attemptId);
    AppAttemptRemovedSchedulerEvent attemptRemovedEvent = new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);
    // Remove the app attempt
    scheduler.handle(attemptRemovedEvent);
    // Make sure the app attempt is not in the queue and stopped.
    List<ApplicationAttemptId> attemptList = scheduler.getAppsInQueue(testUser);
    assertNotNull("Queue missing", attemptList);
    assertFalse("Attempt should not be in the queue", attemptList.contains(attemptId));
    assertTrue("Attempt should have been stopped", attempt.isStopped());
    // The attempt should still show the original queue info.
    assertTrue("Attempt queue has changed", attempt.getQueue().getName().endsWith(testUser));
    // Now move the app: not using an event since there is none
    // in the scheduler. This should throw.
    scheduler.moveApplication(attemptId.getApplicationId(), "default");
}

Example 2 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testDoubleRemoval.

@Test
public void testDoubleRemoval() throws Exception {
    // convenience var
    String testUser = "user1";
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    ApplicationAttemptId attemptId = createAppAttemptId(1, 1);
    // The placement rule will add the app to the user based queue but the
    // passed in queue must exist.
    AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), testUser, testUser);
    scheduler.handle(appAddedEvent);
    AppAttemptAddedSchedulerEvent attemptAddedEvent = new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
    scheduler.handle(attemptAddedEvent);
    // Get a handle on the attempt.
    FSAppAttempt attempt = scheduler.getSchedulerApp(attemptId);
    AppAttemptRemovedSchedulerEvent attemptRemovedEvent = new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);
    // Make sure the app attempt is in the queue.
    List<ApplicationAttemptId> attemptList = scheduler.getAppsInQueue(testUser);
    assertNotNull("Queue missing", attemptList);
    assertTrue("Attempt should be in the queue", attemptList.contains(attemptId));
    assertFalse("Attempt is stopped", attempt.isStopped());
    // Now remove the app attempt
    scheduler.handle(attemptRemovedEvent);
    // The attempt is not in the queue, and stopped
    attemptList = scheduler.getAppsInQueue(testUser);
    assertFalse("Attempt should not be in the queue", attemptList.contains(attemptId));
    assertTrue("Attempt should have been stopped", attempt.isStopped());
    // Now remove the app attempt again, since it is stopped nothing happens.
    scheduler.handle(attemptRemovedEvent);
    // The attempt should still show the original queue info.
    assertTrue("Attempt queue has changed", attempt.getQueue().getName().endsWith(testUser));
}

Example 3 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class TestLeafQueue method testAppAttemptMetrics.

@Test
public void testAppAttemptMetrics() throws Exception {
    // Manipulate queue 'a'
    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));
    // Users
    final String user_0 = "user_0";
    // Submit applications
    final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 1);
    AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(), a.getQueueName(), user_0);
    cs.handle(addAppEvent);
    AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
    cs.handle(addAttemptEvent);
    AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FAILED, false);
    cs.handle(event);
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    // Attempt the same application again
    final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(0, 2);
    FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, null, spyRMContext);
    app_1.setAMResource(Resource.newInstance(100, 1));
    // same user
    a.submitApplicationAttempt(app_1, user_0);
    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(1, a.getMetrics().getAppsPending());
    assertEquals(1, a.getUser(user_0).getActiveApplications());
    assertEquals(app_1.getAMResource().getMemorySize(), a.getMetrics().getUsedAMResourceMB());
    assertEquals(app_1.getAMResource().getVirtualCores(), a.getMetrics().getUsedAMResourceVCores());
    event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FINISHED, false);
    cs.handle(event);
    AppRemovedSchedulerEvent rEvent = new AppRemovedSchedulerEvent(appAttemptId_0.getApplicationId(), RMAppState.FINISHED);
    cs.handle(rEvent);
    assertEquals(1, a.getMetrics().getAppsSubmitted());
    assertEquals(0, a.getMetrics().getAppsPending());
    assertEquals(0, a.getMetrics().getAppsFailed());
    assertEquals(1, a.getMetrics().getAppsCompleted());
    QueueMetrics userMetrics = a.getMetrics().getUserMetrics(user_0);
    assertEquals(1, userMetrics.getAppsSubmitted());
}

Example 4 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class TestFifoScheduler method testBlackListNodes.

@Test(timeout = 50000)
public void testBlackListNodes() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    int rack_num_0 = 0;
    int rack_num_1 = 1;
    // Add 4 nodes in 2 racks
    // host_0_0 in rack0
    String host_0_0 = "127.0.0.1";
    RMNode n1 = MockNodes.newNodeInfo(rack_num_0, MockNodes.newResource(4 * GB), 1, host_0_0);
    fs.handle(new NodeAddedSchedulerEvent(n1));
    // host_0_1 in rack0
    String host_0_1 = "127.0.0.2";
    RMNode n2 = MockNodes.newNodeInfo(rack_num_0, MockNodes.newResource(4 * GB), 1, host_0_1);
    fs.handle(new NodeAddedSchedulerEvent(n2));
    // host_1_0 in rack1
    String host_1_0 = "127.0.0.3";
    RMNode n3 = MockNodes.newNodeInfo(rack_num_1, MockNodes.newResource(4 * GB), 1, host_1_0);
    fs.handle(new NodeAddedSchedulerEvent(n3));
    // host_1_1 in rack1
    String host_1_1 = "127.0.0.4";
    RMNode n4 = MockNodes.newNodeInfo(rack_num_1, MockNodes.newResource(4 * GB), 1, host_1_1);
    fs.handle(new NodeAddedSchedulerEvent(n4));
    // Add one application
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1);
    createMockRMApp(appAttemptId1, rm.getRMContext());
    SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
    fs.handle(appEvent);
    SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    fs.handle(attemptEvent);
    List<ContainerId> emptyId = new ArrayList<ContainerId>();
    List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();
    // Allow rack-locality for rack_1, but blacklist host_1_0
    // Set up resource requests
    // Ask for a 1 GB container for app 1
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "rack1", BuilderUtils.newResource(GB, 1), 1, RMNodeLabelsManager.NO_LABEL));
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1, RMNodeLabelsManager.NO_LABEL));
    fs.allocate(appAttemptId1, ask1, emptyId, Collections.singletonList(host_1_0), null, NULL_UPDATE_REQUESTS);
    // Trigger container assignment
    fs.handle(new NodeUpdateSchedulerEvent(n3));
    // Get the allocation for the application and verify no allocation on
    // blacklist node
    Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation1", 0, allocation1.getContainers().size());
    // verify host_1_1 can get allocated as not in blacklist
    fs.handle(new NodeUpdateSchedulerEvent(n4));
    Allocation allocation2 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation2", 1, allocation2.getContainers().size());
    List<Container> containerList = allocation2.getContainers();
    for (Container container : containerList) {
        Assert.assertEquals("Container is allocated on n4", container.getNodeId(), n4.getNodeID());
    }
    // Ask for a 1 GB container again for app 1
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    // this time, rack0 is also in blacklist, so only host_1_1 is available to
    // be assigned
    ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
    fs.allocate(appAttemptId1, ask2, emptyId, Collections.singletonList("rack0"), null, NULL_UPDATE_REQUESTS);
    // verify n1 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n1));
    Allocation allocation3 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation3", 0, allocation3.getContainers().size());
    // verify n2 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n2));
    Allocation allocation4 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation4", 0, allocation4.getContainers().size());
    // verify n3 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n3));
    Allocation allocation5 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation5", 0, allocation5.getContainers().size());
    fs.handle(new NodeUpdateSchedulerEvent(n4));
    Allocation allocation6 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation6", 1, allocation6.getContainers().size());
    containerList = allocation6.getContainers();
    for (Container container : containerList) {
        Assert.assertEquals("Container is allocated on n4", container.getNodeId(), n4.getNodeID());
    }
    rm.stop();
}

Example 5 with AppAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent in project hadoop by apache.

the class TestFifoScheduler method testNodeLocalAssignment.

@Test(timeout = 2000)
public void testNodeLocalAssignment() throws Exception {
    AsyncDispatcher dispatcher = new InlineDispatcher();
    Configuration conf = new Configuration();
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.rollMasterKey();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    FifoScheduler scheduler = new FifoScheduler();
    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
    scheduler.setRMContext(rmContext);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(new Configuration(), rmContext);
    RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(1024 * 64), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
    scheduler.handle(nodeEvent1);
    int _appId = 1;
    int _appAttemptId = 1;
    ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId);
    createMockRMApp(appAttemptId, rmContext);
    AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1", "user1");
    scheduler.handle(appEvent);
    AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);
    int memory = 64;
    int nConts = 3;
    int priority = 20;
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory, node0.getHostName(), priority, nConts);
    ResourceRequest rackLocal = createResourceRequest(memory, node0.getRackName(), priority, nConts);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority, nConts);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
    // Before the node update event, there are 3 local requests outstanding
    Assert.assertEquals(3, nodeLocal.getNumContainers());
    scheduler.handle(node0Update);
    // After the node update event, check that there are no more local requests
    // outstanding
    Assert.assertEquals(0, nodeLocal.getNumContainers());
    //Also check that the containers were scheduled
    SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
    Assert.assertEquals(3, info.getLiveContainers().size());
    scheduler.stop();
}