Examples with ResourceScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler used on opensource projects

Example 26 with ResourceScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler in project hadoop by apache.

the class TestApplicationPriority method testOrderOfActivatingThePriorityApplicationOnRMRestart.

/**
   * <p>
   * Test case verifies the order of applications activated after RM Restart.
   * </p>
   * <li>App-1 and app-2 submitted and scheduled and running with a priority
   * 5 and 6 Respectively</li>
   * <li>App-3 submitted and scheduled with a priority 7. This
   * is not activated since AMResourceLimit is reached</li>
   * <li>RM restarted</li>
   * <li>App-1 get activated nevertheless of AMResourceLimit</li>
   * <li>App-2 and app-3 put in pendingOrderingPolicy</li>
   * <li>After NM registration, app-3 is activated</li>
   * <p>
   * Expected Output : App-2 must get activated since app-2 was running earlier
   * </p>
   * @throws Exception
   */
@Test
public void testOrderOfActivatingThePriorityApplicationOnRMRestart() throws Exception {
    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
    conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    final DrainDispatcher dispatcher = new DrainDispatcher();
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    MockRM rm1 = new MockRM(conf, memStore) {

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 16384, rm1.getResourceTrackerService());
    nm1.registerNode();
    dispatcher.await();
    ResourceScheduler scheduler = rm1.getRMContext().getScheduler();
    LeafQueue defaultQueue = (LeafQueue) ((CapacityScheduler) scheduler).getQueue("default");
    int memory = (int) (defaultQueue.getAMResourceLimit().getMemorySize() / 2);
    // App-1 with priority 5 submitted and running
    Priority appPriority1 = Priority.newInstance(5);
    RMApp app1 = rm1.submitApp(memory, appPriority1);
    MockAM am1 = MockRM.launchAM(app1, rm1, nm1);
    am1.registerAppAttempt();
    // App-2 with priority 6 submitted and running
    Priority appPriority2 = Priority.newInstance(6);
    RMApp app2 = rm1.submitApp(memory, appPriority2);
    MockAM am2 = MockRM.launchAM(app2, rm1, nm1);
    am2.registerAppAttempt();
    dispatcher.await();
    Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
    Assert.assertEquals(0, defaultQueue.getNumPendingApplications());
    // App-3 with priority 7 submitted and scheduled. But not activated since
    // AMResourceLimit threshold
    Priority appPriority3 = Priority.newInstance(7);
    RMApp app3 = rm1.submitApp(memory, appPriority3);
    dispatcher.await();
    Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
    Assert.assertEquals(1, defaultQueue.getNumPendingApplications());
    Iterator<FiCaSchedulerApp> iterator = defaultQueue.getOrderingPolicy().getSchedulableEntities().iterator();
    FiCaSchedulerApp fcApp2 = iterator.next();
    Assert.assertEquals(app2.getCurrentAppAttempt().getAppAttemptId(), fcApp2.getApplicationAttemptId());
    FiCaSchedulerApp fcApp1 = iterator.next();
    Assert.assertEquals(app1.getCurrentAppAttempt().getAppAttemptId(), fcApp1.getApplicationAttemptId());
    iterator = defaultQueue.getPendingApplications().iterator();
    FiCaSchedulerApp fcApp3 = iterator.next();
    Assert.assertEquals(app3.getCurrentAppAttempt().getAppAttemptId(), fcApp3.getApplicationAttemptId());
    final DrainDispatcher dispatcher1 = new DrainDispatcher();
    // create new RM to represent restart and recover state
    MockRM rm2 = new MockRM(conf, memStore) {

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher1;
        }
    };
    // start new RM
    rm2.start();
    // change NM to point to new RM
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    // Verify RM Apps after this restart
    Assert.assertEquals(3, rm2.getRMContext().getRMApps().size());
    dispatcher1.await();
    scheduler = rm2.getRMContext().getScheduler();
    defaultQueue = (LeafQueue) ((CapacityScheduler) scheduler).getQueue("default");
    // wait for all applications to get added to scheduler
    int count = 50;
    while (count-- > 0) {
        if (defaultQueue.getNumPendingApplications() == 3) {
            break;
        }
        Thread.sleep(50);
    }
    // Before NM registration, AMResourceLimit threshold is 0. So no
    // applications get activated.
    Assert.assertEquals(0, defaultQueue.getNumActiveApplications());
    Assert.assertEquals(3, defaultQueue.getNumPendingApplications());
    // NM resync to new RM
    nm1.registerNode();
    dispatcher1.await();
    // wait for activating applications
    count = 50;
    while (count-- > 0) {
        if (defaultQueue.getNumActiveApplications() == 2) {
            break;
        }
        Thread.sleep(50);
    }
    Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
    Assert.assertEquals(1, defaultQueue.getNumPendingApplications());
    // verify for order of activated applications iterator
    iterator = defaultQueue.getOrderingPolicy().getSchedulableEntities().iterator();
    fcApp2 = iterator.next();
    Assert.assertEquals(app2.getCurrentAppAttempt().getAppAttemptId(), fcApp2.getApplicationAttemptId());
    fcApp1 = iterator.next();
    Assert.assertEquals(app1.getCurrentAppAttempt().getAppAttemptId(), fcApp1.getApplicationAttemptId());
    // verify for pending application iterator. It should be app-3 attempt
    iterator = defaultQueue.getPendingApplications().iterator();
    fcApp3 = iterator.next();
    Assert.assertEquals(app3.getCurrentAppAttempt().getAppAttemptId(), fcApp3.getApplicationAttemptId());
    rm2.stop();
    rm1.stop();
}

Example 27 with ResourceScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler in project hadoop by apache.

the class TestCapacityScheduler method testMoveAppSuccess.

@Test
public void testMoveAppSuccess() throws Exception {
    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
    // Register node1
    String host_0 = "host_0";
    NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(5 * GB, 1));
    // Register node2
    String host_1 = "host_1";
    NodeManager nm_1 = registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(5 * GB, 1));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit application_0
    Application application_0 = new Application("user_0", "a1", resourceManager);
    // app + app attempt event sent to scheduler
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    application_0.addNodeManager(host_1, 1234, nm_1);
    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0, host_1 });
    application_0.addTask(task_0_0);
    // Submit application_1
    Application application_1 = new Application("user_1", "b2", resourceManager);
    // app + app attempt event sent to scheduler
    application_1.submit();
    application_1.addNodeManager(host_0, 1234, nm_0);
    application_1.addNodeManager(host_1, 1234, nm_1);
    Resource capability_1_0 = Resources.createResource(1 * GB, 1);
    application_1.addResourceRequestSpec(priority_1, capability_1_0);
    Resource capability_1_1 = Resources.createResource(2 * GB, 1);
    application_1.addResourceRequestSpec(priority_0, capability_1_1);
    Task task_1_0 = new Task(application_1, priority_1, new String[] { host_0, host_1 });
    application_1.addTask(task_1_0);
    // Send resource requests to the scheduler
    // allocate
    application_0.schedule();
    // allocate
    application_1.schedule();
    // b2 can only run 1 app at a time
    scheduler.moveApplication(application_0.getApplicationId(), "b2");
    nodeUpdate(nm_0);
    nodeUpdate(nm_1);
    // Get allocations from the scheduler
    // task_0_0
    application_0.schedule();
    checkApplicationResourceUsage(0 * GB, application_0);
    // task_1_0
    application_1.schedule();
    checkApplicationResourceUsage(1 * GB, application_1);
    // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is
    // not scheduled
    checkNodeResourceUsage(1 * GB, nm_0);
    checkNodeResourceUsage(0 * GB, nm_1);
    // lets move application_0 to a queue where it can run
    scheduler.moveApplication(application_0.getApplicationId(), "a2");
    application_0.schedule();
    nodeUpdate(nm_1);
    // Get allocations from the scheduler
    // task_0_0
    application_0.schedule();
    checkApplicationResourceUsage(3 * GB, application_0);
    checkNodeResourceUsage(1 * GB, nm_0);
    checkNodeResourceUsage(3 * GB, nm_1);
}

Example 28 with ResourceScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler in project hadoop by apache.

the class TestCapacityScheduler method testConfValidation.

@Test(timeout = 30000)
public void testConfValidation() throws Exception {
    ResourceScheduler scheduler = new CapacityScheduler();
    scheduler.setRMContext(resourceManager.getRMContext());
    Configuration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
    try {
        scheduler.reinitialize(conf, mockContext);
        fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation.");
    } catch (YarnRuntimeException e) {
        // Exception is expected.
        assertTrue("The thrown exception is not the expected one.", e.getMessage().startsWith("Invalid resource scheduler memory"));
    }
    conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1);
    try {
        scheduler.reinitialize(conf, mockContext);
        fail("Exception is expected because the min vcores allocation is" + " larger than the max vcores allocation.");
    } catch (YarnRuntimeException e) {
        // Exception is expected.
        assertTrue("The thrown exception is not the expected one.", e.getMessage().startsWith("Invalid resource scheduler vcores"));
    }
}

Example 29 with ResourceScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler in project hadoop by apache.

the class TestCapacityScheduler method testMoveAppViolateQueueState.

@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
    resourceManager = new ResourceManager() {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
            mgr.init(getConfig());
            return mgr;
        }
    };
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(csConf);
    StringBuilder qState = new StringBuilder();
    qState.append(CapacitySchedulerConfiguration.PREFIX).append(B).append(CapacitySchedulerConfiguration.DOT).append(CapacitySchedulerConfiguration.STATE);
    csConf.set(qState.toString(), QueueState.STOPPED.name());
    YarnConfiguration conf = new YarnConfiguration(csConf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    resourceManager.init(conf);
    resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
    resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
    ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
    mockContext = mock(RMContext.class);
    when(mockContext.getConfigurationProvider()).thenReturn(new LocalConfigurationProvider());
    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
    // Register node1
    String host_0 = "host_0";
    NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(6 * GB, 1));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit application_0
    Application application_0 = new Application("user_0", "a1", resourceManager);
    // app + app attempt event sent to scheduler
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    // allocate
    application_0.schedule();
    // task_0_0 allocated
    nodeUpdate(nm_0);
    // Get allocations from the scheduler
    // task_0_0
    application_0.schedule();
    checkApplicationResourceUsage(3 * GB, application_0);
    checkNodeResourceUsage(3 * GB, nm_0);
    // b2 queue contains 3GB consumption app,
    // add another 3GB will hit max capacity limit on queue b
    scheduler.moveApplication(application_0.getApplicationId(), "b1");
}

Example 30 with ResourceScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler in project hadoop by apache.

the class TestRMWebServicesNodes method verifyNodeInfoGeneric.

public void verifyNodeInfoGeneric(RMNode node, String state, String rack, String id, String nodeHostName, String nodeHTTPAddress, long lastHealthUpdate, String healthReport, int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores, long availVirtualCores, String version, int nodePhysicalMemoryMB, int nodeVirtualMemoryMB, double nodeCPUUsage, int containersPhysicalMemoryMB, int containersVirtualMemoryMB, double containersCPUUsage, int numRunningOpportContainers, long usedMemoryOpportGB, int usedVirtualCoresOpport, int numQueuedContainers) throws JSONException, Exception {
    ResourceScheduler sched = rm.getResourceScheduler();
    SchedulerNodeReport report = sched.getNodeReport(node.getNodeID());
    OpportunisticContainersStatus opportunisticStatus = node.getOpportunisticContainersStatus();
    WebServicesTestUtils.checkStringMatch("state", node.getState().toString(), state);
    WebServicesTestUtils.checkStringMatch("rack", node.getRackName(), rack);
    WebServicesTestUtils.checkStringMatch("id", node.getNodeID().toString(), id);
    WebServicesTestUtils.checkStringMatch("nodeHostName", node.getNodeID().getHost(), nodeHostName);
    WebServicesTestUtils.checkStringMatch("healthReport", String.valueOf(node.getHealthReport()), healthReport);
    String expectedHttpAddress = node.getNodeID().getHost() + ":" + node.getHttpPort();
    WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", expectedHttpAddress, nodeHTTPAddress);
    WebServicesTestUtils.checkStringMatch("version", node.getNodeManagerVersion(), version);
    if (node.getNodeUtilization() != null) {
        ResourceUtilization nodeResource = ResourceUtilization.newInstance(nodePhysicalMemoryMB, nodeVirtualMemoryMB, (float) nodeCPUUsage);
        assertEquals("nodeResourceUtilization doesn't match", node.getNodeUtilization(), nodeResource);
    }
    if (node.getAggregatedContainersUtilization() != null) {
        ResourceUtilization containerResource = ResourceUtilization.newInstance(containersPhysicalMemoryMB, containersVirtualMemoryMB, (float) containersCPUUsage);
        assertEquals("containerResourceUtilization doesn't match", node.getAggregatedContainersUtilization(), containerResource);
    }
    long expectedHealthUpdate = node.getLastHealthReportTime();
    assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate + " expected: " + expectedHealthUpdate, expectedHealthUpdate, lastHealthUpdate);
    if (report != null) {
        assertEquals("numContainers doesn't match: " + numContainers, report.getNumContainers(), numContainers);
        assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report.getUsedResource().getMemorySize(), usedMemoryMB);
        assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report.getAvailableResource().getMemorySize(), availMemoryMB);
        assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report.getUsedResource().getVirtualCores(), usedVirtualCores);
        assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report.getAvailableResource().getVirtualCores(), availVirtualCores);
    }
    if (opportunisticStatus != null) {
        assertEquals("numRunningOpportContainers doesn't match: " + numRunningOpportContainers, opportunisticStatus.getRunningOpportContainers(), numRunningOpportContainers);
        assertEquals("usedMemoryOpportGB doesn't match: " + usedMemoryOpportGB, opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpportGB);
        assertEquals("usedVirtualCoresOpport doesn't match: " + usedVirtualCoresOpport, opportunisticStatus.getOpportCoresUsed(), usedVirtualCoresOpport);
        assertEquals("numQueuedContainers doesn't match: " + numQueuedContainers, opportunisticStatus.getQueuedOpportContainers(), numQueuedContainers);
    }
}