Examples with AppAttemptAddedSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent used on opensource projects

Example 21 with AppAttemptAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent in project hadoop by apache.

the class TestFifoScheduler method testHeadroom.

@Test(timeout = 50000)
public void testHeadroom() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    // Add a node
    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
    fs.handle(new NodeAddedSchedulerEvent(n1));
    // Add two applications
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1);
    createMockRMApp(appAttemptId1, rm.getRMContext());
    SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
    fs.handle(appEvent);
    SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    fs.handle(attemptEvent);
    ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(appId2, 1);
    createMockRMApp(appAttemptId2, rm.getRMContext());
    SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user");
    fs.handle(appEvent2);
    SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
    fs.handle(attemptEvent2);
    List<ContainerId> emptyId = new ArrayList<ContainerId>();
    List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();
    // Set up resource requests
    // Ask for a 1 GB container for app 1
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
    fs.allocate(appAttemptId1, ask1, emptyId, null, null, NULL_UPDATE_REQUESTS);
    // Ask for a 2 GB container for app 2
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
    fs.allocate(appAttemptId2, ask2, emptyId, null, null, NULL_UPDATE_REQUESTS);
    // Trigger container assignment
    fs.handle(new NodeUpdateSchedulerEvent(n1));
    // Get the allocation for the applications and verify headroom
    Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("Allocation headroom", 1 * GB, allocation1.getResourceLimit().getMemorySize());
    Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("Allocation headroom", 1 * GB, allocation2.getResourceLimit().getMemorySize());
    rm.stop();
}

Example 22 with AppAttemptAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent in project hadoop by apache.

the class TestFifoScheduler method testAppAttemptMetrics.

@Test(timeout = 5000)
public void testAppAttemptMetrics() throws Exception {
    AsyncDispatcher dispatcher = new InlineDispatcher();
    FifoScheduler scheduler = new FifoScheduler();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, scheduler);
    ((RMContextImpl) rmContext).setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    Configuration conf = new Configuration();
    ((RMContextImpl) rmContext).setScheduler(scheduler);
    scheduler.setRMContext(rmContext);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, rmContext);
    QueueMetrics metrics = scheduler.getRootQueueMetrics();
    int beforeAppsSubmitted = metrics.getAppsSubmitted();
    ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
    scheduler.handle(appEvent);
    SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);
    appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
    SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent2);
    int afterAppsSubmitted = metrics.getAppsSubmitted();
    Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
    scheduler.stop();
}

Example 23 with AppAttemptAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent in project hadoop by apache.

the class TestFifoScheduler method testUpdateResourceOnNode.

@Test(timeout = 2000)
public void testUpdateResourceOnNode() throws Exception {
    AsyncDispatcher dispatcher = new InlineDispatcher();
    Configuration conf = new Configuration();
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.rollMasterKey();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    FifoScheduler scheduler = new FifoScheduler();
    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
    NullRMNodeLabelsManager nlm = new NullRMNodeLabelsManager();
    nlm.init(new Configuration());
    rmContext.setNodeLabelManager(nlm);
    scheduler.setRMContext(rmContext);
    ((RMContextImpl) rmContext).setScheduler(scheduler);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(new Configuration(), rmContext);
    RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 4), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
    scheduler.handle(nodeEvent1);
    assertEquals(scheduler.getNumClusterNodes(), 1);
    Resource newResource = Resources.createResource(1024, 4);
    NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(newResource, ResourceOption.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
    scheduler.handle(node0ResourceUpdate);
    // SchedulerNode's total resource and available resource are changed.
    assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).getTotalResource().getMemorySize());
    assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).getUnallocatedResource().getMemorySize(), 1024);
    QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
    Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
    int _appId = 1;
    int _appAttemptId = 1;
    ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId);
    createMockRMApp(appAttemptId, rmContext);
    AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1", "user1");
    scheduler.handle(appEvent);
    AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);
    int memory = 1024;
    int priority = 1;
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory, node0.getHostName(), priority, 1);
    ResourceRequest rackLocal = createResourceRequest(memory, node0.getRackName(), priority, 1);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority, 1);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    // Before the node update event, there are one local request
    Assert.assertEquals(1, nodeLocal.getNumContainers());
    NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
    // Now schedule.
    scheduler.handle(node0Update);
    // After the node update event, check no local request
    Assert.assertEquals(0, nodeLocal.getNumContainers());
    // Also check that one container was scheduled
    SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
    Assert.assertEquals(1, info.getLiveContainers().size());
    // And check the default Queue now is full.
    queueInfo = scheduler.getQueueInfo(null, false, false);
    Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}