Examples with DominantResourceFairnessPolicy org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy used on opensource projects

Example 1 with DominantResourceFairnessPolicy

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy in project hadoop by apache.

the class TestFairScheduler method testDRFHierarchicalQueues.

@Test
public void testDRFHierarchicalQueues() throws Exception {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(12288, 12), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
    scheduler.handle(nodeEvent);
    ApplicationAttemptId appAttId1 = createSchedulingRequest(3074, 1, "queue1.subqueue1", "user1", 2);
    // so that start times will be different
    Thread.sleep(3);
    FSAppAttempt app1 = scheduler.getSchedulerApp(appAttId1);
    ApplicationAttemptId appAttId2 = createSchedulingRequest(1024, 3, "queue1.subqueue1", "user1", 2);
    // so that start times will be different
    Thread.sleep(3);
    FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2);
    ApplicationAttemptId appAttId3 = createSchedulingRequest(2048, 2, "queue1.subqueue2", "user1", 2);
    // so that start times will be different
    Thread.sleep(3);
    FSAppAttempt app3 = scheduler.getSchedulerApp(appAttId3);
    ApplicationAttemptId appAttId4 = createSchedulingRequest(1024, 2, "queue2", "user1", 2);
    // so that start times will be different
    Thread.sleep(3);
    FSAppAttempt app4 = scheduler.getSchedulerApp(appAttId4);
    DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
    drfPolicy.initialize(scheduler.getContext());
    scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy);
    scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
    scheduler.getQueueManager().getQueue("queue1.subqueue1").setPolicy(drfPolicy);
    scheduler.update();
    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
    scheduler.handle(updateEvent);
    // app1 gets first container because it asked first
    Assert.assertEquals(1, app1.getLiveContainers().size());
    scheduler.handle(updateEvent);
    // app4 gets second container because it's on queue2
    Assert.assertEquals(1, app4.getLiveContainers().size());
    scheduler.handle(updateEvent);
    // app4 gets another container because queue2's dominant share of memory
    // is still less than queue1's of cpu
    Assert.assertEquals(2, app4.getLiveContainers().size());
    scheduler.handle(updateEvent);
    // app3 gets one because queue1 gets one and queue1.subqueue2 is behind
    // queue1.subqueue1
    Assert.assertEquals(1, app3.getLiveContainers().size());
    scheduler.handle(updateEvent);
    // app4 would get another one, but it doesn't have any requests
    // queue1.subqueue2 is still using less than queue1.subqueue1, so it
    // gets another
    Assert.assertEquals(2, app3.getLiveContainers().size());
    // queue1.subqueue1 is behind again, so it gets one, which it gives to app2
    scheduler.handle(updateEvent);
    Assert.assertEquals(1, app2.getLiveContainers().size());
    // at this point, we've used all our CPU up, so nobody else should get a container
    scheduler.handle(updateEvent);
    Assert.assertEquals(1, app1.getLiveContainers().size());
    Assert.assertEquals(1, app2.getLiveContainers().size());
    Assert.assertEquals(2, app3.getLiveContainers().size());
    Assert.assertEquals(2, app4.getLiveContainers().size());
}

Example 2 with DominantResourceFairnessPolicy

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy in project hadoop by apache.

the class TestWorkPreservingRMRestart method checkFSQueue.

private void checkFSQueue(ResourceManager rm, SchedulerApplication schedulerApp, Resource usedResources, Resource availableResources) throws Exception {
    // waiting for RM's scheduling apps
    int retry = 0;
    Resource assumedFairShare = Resource.newInstance(8192, 8);
    while (true) {
        Thread.sleep(100);
        if (assumedFairShare.equals(((FairScheduler) rm.getResourceScheduler()).getQueueManager().getRootQueue().getFairShare())) {
            break;
        }
        retry++;
        if (retry > 30) {
            Assert.fail("Apps are not scheduled within assumed timeout");
        }
    }
    FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
    FSParentQueue root = scheduler.getQueueManager().getRootQueue();
    // ************ check cluster used Resources ********
    assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy);
    assertEquals(usedResources, root.getResourceUsage());
    // ************ check app headroom ****************
    FSAppAttempt schedulerAttempt = (FSAppAttempt) schedulerApp.getCurrentAppAttempt();
    assertEquals(availableResources, schedulerAttempt.getHeadroom());
    // ************ check queue metrics ****************
    QueueMetrics queueMetrics = scheduler.getRootQueueMetrics();
    assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemorySize(), availableResources.getVirtualCores(), usedResources.getMemorySize(), usedResources.getVirtualCores());
}

Example 3 with DominantResourceFairnessPolicy

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy in project hadoop by apache.

the class TestSchedulingPolicy method testPolicyReinitilization.

@Test
public void testPolicyReinitilization() throws IOException {
    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("<schedulingPolicy>fair</schedulingPolicy>");
    out.println("    <queue name=\"child1\">");
    out.println("    <schedulingPolicy>fair</schedulingPolicy>");
    out.println("    </queue>");
    out.println("    <queue name=\"child2\">");
    out.println("    <schedulingPolicy>fair</schedulingPolicy>");
    out.println("    </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
    scheduler.init(conf);
    // Set child1 to 'drf' which is not allowed, then reload the allocation file
    out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("<schedulingPolicy>fair</schedulingPolicy>");
    out.println("    <queue name=\"child1\">");
    out.println("    <schedulingPolicy>drf</schedulingPolicy>");
    out.println("    </queue>");
    out.println("    <queue name=\"child2\">");
    out.println("    <schedulingPolicy>fifo</schedulingPolicy>");
    out.println("    </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
    scheduler.reinitialize(conf, null);
    FSQueue child1 = scheduler.getQueueManager().getQueue("child1");
    assertTrue("Queue 'child1' should still be 'fair' since 'drf' isn't allowed" + " if its parent policy is 'fair'.", child1.getPolicy() instanceof FairSharePolicy);
    FSQueue child2 = scheduler.getQueueManager().getQueue("child2");
    assertTrue("Queue 'child2' should still be 'fair' there is a policy" + " violation while reinitialization.", child2.getPolicy() instanceof FairSharePolicy);
    // Set both child1 and root to 'drf', then reload the allocation file
    out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("<schedulingPolicy>drf</schedulingPolicy>");
    out.println("    <queue name=\"child1\">");
    out.println("    <schedulingPolicy>drf</schedulingPolicy>");
    out.println("    </queue>");
    out.println("    <queue name=\"child2\">");
    out.println("    <schedulingPolicy>fifo</schedulingPolicy>");
    out.println("    </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
    scheduler.reinitialize(conf, null);
    child1 = scheduler.getQueueManager().getQueue("child1");
    assertTrue("Queue 'child1' should be 'drf' since both 'root' and 'child1'" + " are 'drf'.", child1.getPolicy() instanceof DominantResourceFairnessPolicy);
    child2 = scheduler.getQueueManager().getQueue("child2");
    assertTrue("Queue 'child2' should still be 'fifo' there is no policy" + " violation while reinitialization.", child2.getPolicy() instanceof FifoPolicy);
}

Example 4 with DominantResourceFairnessPolicy

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy in project hadoop by apache.

the class TestFairScheduler method testBasicDRFWithQueues.

/**
   * Two apps on one queue, one app on another
   */
@Test
public void testBasicDRFWithQueues() throws Exception {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 7), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
    scheduler.handle(nodeEvent);
    ApplicationAttemptId appAttId1 = createSchedulingRequest(3072, 1, "queue1", "user1", 2);
    FSAppAttempt app1 = scheduler.getSchedulerApp(appAttId1);
    ApplicationAttemptId appAttId2 = createSchedulingRequest(2048, 2, "queue1", "user1", 2);
    FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2);
    ApplicationAttemptId appAttId3 = createSchedulingRequest(1024, 2, "queue2", "user1", 2);
    FSAppAttempt app3 = scheduler.getSchedulerApp(appAttId3);
    DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
    drfPolicy.initialize(scheduler.getContext());
    scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy);
    scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
    scheduler.update();
    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
    scheduler.handle(updateEvent);
    Assert.assertEquals(1, app1.getLiveContainers().size());
    scheduler.handle(updateEvent);
    Assert.assertEquals(1, app3.getLiveContainers().size());
    scheduler.handle(updateEvent);
    Assert.assertEquals(2, app3.getLiveContainers().size());
    scheduler.handle(updateEvent);
    Assert.assertEquals(1, app2.getLiveContainers().size());
}

Example 5 with DominantResourceFairnessPolicy

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy in project hadoop by apache.

the class TestFSAppAttempt method testHeadroom.

@Test
public void testHeadroom() {
    final FairScheduler mockScheduler = Mockito.mock(FairScheduler.class);
    Mockito.when(mockScheduler.getClock()).thenReturn(scheduler.getClock());
    final FSLeafQueue mockQueue = Mockito.mock(FSLeafQueue.class);
    final Resource queueMaxResources = Resource.newInstance(5 * 1024, 3);
    final Resource queueFairShare = Resources.createResource(4096, 2);
    final Resource queueUsage = Resource.newInstance(2048, 2);
    final Resource queueStarvation = Resources.subtract(queueFairShare, queueUsage);
    final Resource queueMaxResourcesAvailable = Resources.subtract(queueMaxResources, queueUsage);
    final Resource clusterResource = Resources.createResource(8192, 8);
    final Resource clusterUsage = Resources.createResource(2048, 2);
    final Resource clusterAvailable = Resources.subtract(clusterResource, clusterUsage);
    final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class);
    Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources);
    Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare);
    Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
    Mockito.when(mockScheduler.getClusterResource()).thenReturn(clusterResource);
    Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn(clusterUsage);
    Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn(fakeRootQueueMetrics);
    ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
    RMContext rmContext = resourceManager.getRMContext();
    FSAppAttempt schedulerApp = new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue, null, rmContext);
    // Min of Memory and CPU across cluster and queue is used in
    // DominantResourceFairnessPolicy
    Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy.getInstance(DominantResourceFairnessPolicy.class));
    verifyHeadroom(schedulerApp, min(queueStarvation.getMemorySize(), clusterAvailable.getMemorySize(), queueMaxResourcesAvailable.getMemorySize()), min(queueStarvation.getVirtualCores(), clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()));
    // Fair and Fifo ignore CPU of queue, so use cluster available CPU
    Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy.getInstance(FairSharePolicy.class));
    verifyHeadroom(schedulerApp, min(queueStarvation.getMemorySize(), clusterAvailable.getMemorySize(), queueMaxResourcesAvailable.getMemorySize()), Math.min(clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()));
    Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy.getInstance(FifoPolicy.class));
    verifyHeadroom(schedulerApp, min(queueStarvation.getMemorySize(), clusterAvailable.getMemorySize(), queueMaxResourcesAvailable.getMemorySize()), Math.min(clusterAvailable.getVirtualCores(), queueMaxResourcesAvailable.getVirtualCores()));
}