Examples with Priority org.apache.hadoop.yarn.api.records.Priority used on opensource projects

Example 36 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class TestSchedulerHealth method testCapacitySchedulerAllocation.

@Test
public void testCapacitySchedulerAllocation() throws Exception {
    setup();
    boolean isCapacityScheduler = resourceManager.getResourceScheduler() instanceof CapacityScheduler;
    assumeTrue("This test is only supported on Capacity Scheduler", isCapacityScheduler);
    // Register node1
    String host_0 = "host_0";
    NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(5 * 1024, 1));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit an application
    Application application_0 = new Application("user_0", "default", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(1024, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * 1024, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0 });
    application_0.addTask(task_0_0);
    Task task_0_1 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_1);
    // Send resource requests to the scheduler
    application_0.schedule();
    // Send a heartbeat to kick the tires on the Scheduler
    nodeUpdate(nm_0);
    SchedulerHealth sh = ((CapacityScheduler) resourceManager.getResourceScheduler()).getSchedulerHealth();
    // Now SchedulerHealth records last container allocated, aggregated
    // allocation account will not be changed
    Assert.assertEquals(1, sh.getAllocationCount().longValue());
    Assert.assertEquals(Resource.newInstance(1 * 1024, 1), sh.getResourcesAllocated());
    Assert.assertEquals(2, sh.getAggregateAllocationCount().longValue());
    Assert.assertEquals("host_0:1234", sh.getLastAllocationDetails().getNodeId().toString());
    Assert.assertEquals("root.default", sh.getLastAllocationDetails().getQueue());
    Task task_0_2 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_2);
    application_0.schedule();
    nodeUpdate(nm_0);
    Assert.assertEquals(1, sh.getAllocationCount().longValue());
    Assert.assertEquals(Resource.newInstance(2 * 1024, 1), sh.getResourcesAllocated());
    Assert.assertEquals(3, sh.getAggregateAllocationCount().longValue());
    Assert.assertEquals("host_0:1234", sh.getLastAllocationDetails().getNodeId().toString());
    Assert.assertEquals("root.default", sh.getLastAllocationDetails().getQueue());
}

Example 37 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class TestSchedulerHealth method testCapacitySchedulerReservation.

@Test
public void testCapacitySchedulerReservation() throws Exception {
    setup();
    boolean isCapacityScheduler = resourceManager.getResourceScheduler() instanceof CapacityScheduler;
    assumeTrue("This test is only supported on Capacity Scheduler", isCapacityScheduler);
    // Register nodes
    String host_0 = "host_0";
    NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(2 * 1024, 1));
    String host_1 = "host_1";
    NodeManager nm_1 = registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(5 * 1024, 1));
    nodeUpdate(nm_0);
    nodeUpdate(nm_1);
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit an application
    Application application_0 = new Application("user_0", "default", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    application_0.addNodeManager(host_1, 1234, nm_1);
    Resource capability_0_0 = Resources.createResource(1024, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * 1024, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    application_0.schedule();
    // Send a heartbeat to kick the tires on the Scheduler
    nodeUpdate(nm_0);
    SchedulerHealth sh = ((CapacityScheduler) resourceManager.getResourceScheduler()).getSchedulerHealth();
    Assert.assertEquals(1, sh.getAllocationCount().longValue());
    Assert.assertEquals(Resource.newInstance(1024, 1), sh.getResourcesAllocated());
    Assert.assertEquals(1, sh.getAggregateAllocationCount().longValue());
    Assert.assertEquals("host_0:1234", sh.getLastAllocationDetails().getNodeId().toString());
    Assert.assertEquals("root.default", sh.getLastAllocationDetails().getQueue());
    Task task_0_1 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_1);
    application_0.schedule();
    nodeUpdate(nm_0);
    Assert.assertEquals(0, sh.getAllocationCount().longValue());
    Assert.assertEquals(1, sh.getReservationCount().longValue());
    Assert.assertEquals(Resource.newInstance(2 * 1024, 1), sh.getResourcesReserved());
    Assert.assertEquals(1, sh.getAggregateAllocationCount().longValue());
    Assert.assertEquals("host_0:1234", sh.getLastAllocationDetails().getNodeId().toString());
    Assert.assertEquals("root.default", sh.getLastAllocationDetails().getQueue());
}

Example 38 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class TestApplicationPriority method testMaxPriorityValidation.

@Test
public void testMaxPriorityValidation() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    // Set Max Application Priority as 10
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    Priority maxPriority = Priority.newInstance(10);
    MockRM rm = new MockRM(conf);
    rm.start();
    Priority appPriority1 = Priority.newInstance(15);
    rm.registerNode("127.0.0.1:1234", 8 * GB);
    RMApp app1 = rm.submitApp(1 * GB, appPriority1);
    // Application submission should be successful and verify priority
    Assert.assertEquals(app1.getApplicationSubmissionContext().getPriority(), maxPriority);
    rm.stop();
}

Example 39 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class TestApplicationPriority method testPriorityWithPendingApplications.

@Test
public void testPriorityWithPendingApplications() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    // Set Max Application Priority as 10
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    MockRM rm = new MockRM(conf);
    rm.start();
    Priority appPriority1 = Priority.newInstance(5);
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 8 * GB);
    RMApp app1 = rm.submitApp(1 * GB, appPriority1);
    // kick the scheduler, 1 GB given to AM1, remaining 7GB on nm1
    MockAM am1 = MockRM.launchAM(app1, rm, nm1);
    am1.registerAppAttempt();
    // kick the scheduler, 7 containers will be allocated for App1
    List<Container> allocated1 = am1.allocateAndWaitForContainers("127.0.0.1", 7, 1 * GB, nm1);
    Assert.assertEquals(7, allocated1.size());
    Assert.assertEquals(1 * GB, allocated1.get(0).getResource().getMemorySize());
    // check node report, 8 GB used (1 AM and 7 containers) and 0 GB available
    SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
    Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize());
    Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
    // Submit the second app App2 with priority 7
    Priority appPriority2 = Priority.newInstance(7);
    RMApp app2 = rm.submitApp(1 * GB, appPriority2);
    // Submit the third app App3 with priority 8
    Priority appPriority3 = Priority.newInstance(8);
    RMApp app3 = rm.submitApp(1 * GB, appPriority3);
    // Submit the second app App4 with priority 6
    Priority appPriority4 = Priority.newInstance(6);
    RMApp app4 = rm.submitApp(1 * GB, appPriority4);
    // Only one app can run as AM resource limit restricts it. Kill app1,
    // If app3 (highest priority among rest) gets active, it indicates that
    // priority is working with pendingApplications.
    rm.killApp(app1.getApplicationId());
    rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.KILLED);
    // kick the scheduler, app3 (high among pending) gets free space
    MockAM am3 = MockRM.launchAM(app3, rm, nm1);
    am3.registerAppAttempt();
    // check node report, 1 GB used and 7 GB available
    report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
    Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
    Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize());
    rm.stop();
}

Example 40 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class TestApplicationPriority method testUpdateInvalidPriorityAtRuntime.

@Test
public void testUpdateInvalidPriorityAtRuntime() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    // Set Max Application Priority as 10
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    MockRM rm = new MockRM(conf);
    rm.start();
    Priority appPriority1 = Priority.newInstance(5);
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB);
    RMApp app1 = rm.submitApp(1 * GB, appPriority1);
    // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1
    MockAM am1 = MockRM.launchAM(app1, rm, nm1);
    am1.registerAppAttempt();
    // get scheduler
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    // Change the priority of App1 to 15
    Priority appPriority2 = Priority.newInstance(15);
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(app1.getUser());
    cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null, ugi);
    // get scheduler app
    FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications().get(app1.getApplicationId()).getCurrentAppAttempt();
    // Verify whether priority 15 is reset to 10
    Priority appPriority3 = Priority.newInstance(10);
    Assert.assertEquals(appPriority3, schedulerAppAttempt.getPriority());
    rm.stop();
}