Examples with DefaultResourceCalculator org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator used on opensource projects

Example 6 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testMergesubtractTestNonNegative.

@Test
public void testMergesubtractTestNonNegative() throws PlanningException {
    // starting with default array example
    TreeMap<Long, Resource> a = new TreeMap<>();
    TreeMap<Long, Resource> b = new TreeMap<>();
    setupArrays(a, b);
    RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
    RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
    try {
        RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.subtractTestNonNegative, 0, 60);
        fail();
    } catch (PlanningException pe) {
    // Expected!
    }
    // NOTE a is empty!! so the subtraction is implicitly considered negative
    // and the test should fail
    a = new TreeMap<>();
    b = new TreeMap<>();
    b.put(11L, Resource.newInstance(5, 6));
    rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
    rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
    try {
        RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.subtractTestNonNegative, 0, 60);
        fail();
    } catch (PlanningException pe) {
    // Expected!
    }
    // Testing that the subtractTestNonNegative detects problems even if only one
    // of the resource dimensions is "<0"
    a.put(10L, Resource.newInstance(10, 5));
    b.put(11L, Resource.newInstance(5, 6));
    rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
    rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
    try {
        RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.subtractTestNonNegative, 0, 60);
        fail();
    } catch (PlanningException pe) {
    // Expected!
    }
    // try with reverse setting
    a.put(10L, Resource.newInstance(5, 10));
    b.put(11L, Resource.newInstance(6, 5));
    rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
    rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
    try {
        RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.subtractTestNonNegative, 0, 60);
        fail();
    } catch (PlanningException pe) {
    // Expected!
    }
    // trying a case that should work
    a.put(10L, Resource.newInstance(10, 6));
    b.put(11L, Resource.newInstance(5, 6));
    rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
    rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
    RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.subtractTestNonNegative, 0, 60);
}

Example 7 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testPartialRemoval.

@Test
public void testPartialRemoval() {
    ResourceCalculator resCalc = new DefaultResourceCalculator();
    RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc);
    ReservationInterval riAdd = new ReservationInterval(10, 20);
    Resource rr = Resource.newInstance(1024 * 100, 100);
    ReservationInterval riAdd2 = new ReservationInterval(20, 30);
    Resource rr2 = Resource.newInstance(1024 * 200, 200);
    ReservationInterval riRemove = new ReservationInterval(12, 25);
    // same if we use this
    // ReservationRequest rrRemove =
    // ReservationRequest.newInstance(Resource.newInstance(1024, 1), 100, 1,6);
    LOG.info(rleSparseVector.toString());
    rleSparseVector.addInterval(riAdd, rr);
    rleSparseVector.addInterval(riAdd2, rr2);
    LOG.info(rleSparseVector.toString());
    rleSparseVector.removeInterval(riRemove, rr);
    LOG.info(rleSparseVector.toString());
    // Current bug prevents this to pass. The RLESparseResourceAllocation
    // does not handle removal of "partial"
    // allocations correctly.
    Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(10).getMemorySize());
    Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(13).getMemorySize());
    Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(19).getMemorySize());
    Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(21).getMemorySize());
    Assert.assertEquals(2 * 102400, rleSparseVector.getCapacityAtTime(26).getMemorySize());
    ReservationInterval riRemove2 = new ReservationInterval(9, 13);
    rleSparseVector.removeInterval(riRemove2, rr);
    LOG.info(rleSparseVector.toString());
    Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(11).getMemorySize());
    Assert.assertEquals(-102400, rleSparseVector.getCapacityAtTime(9).getMemorySize());
    Assert.assertEquals(0, rleSparseVector.getCapacityAtTime(13).getMemorySize());
    Assert.assertEquals(102400, rleSparseVector.getCapacityAtTime(20).getMemorySize());
}

Example 8 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testMergeAdd.

@Test
public void testMergeAdd() throws PlanningException {
    TreeMap<Long, Resource> a = new TreeMap<>();
    TreeMap<Long, Resource> b = new TreeMap<>();
    setupArrays(a, b);
    RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
    RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
    RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.add, 18, 45);
    System.out.println(out);
    long[] time = { 18, 20, 22, 30, 33, 40, 43, 45 };
    int[] alloc = { 10, 15, 20, 25, 30, 40, 30 };
    validate(out, time, alloc);
}

Example 9 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testMergeMin.

@Test
public void testMergeMin() throws PlanningException {
    TreeMap<Long, Resource> a = new TreeMap<>();
    TreeMap<Long, Resource> b = new TreeMap<>();
    setupArrays(a, b);
    RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
    RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
    RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.min, 0, 60);
    System.out.println(out);
    long[] time = { 10, 22, 33, 40, 43, 50, 60 };
    int[] alloc = { 5, 10, 15, 20, 10, 0 };
    validate(out, time, alloc);
}

Example 10 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestLeafQueue method testApplicationQueuePercent.

@Test
public void testApplicationQueuePercent() throws Exception {
    Resource res = Resource.newInstance(10 * 1024, 10);
    CapacityScheduler scheduler = mock(CapacityScheduler.class);
    when(scheduler.getClusterResource()).thenReturn(res);
    when(scheduler.getResourceCalculator()).thenReturn(new DefaultResourceCalculator());
    ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getEpoch()).thenReturn(3L);
    when(rmContext.getScheduler()).thenReturn(scheduler);
    when(rmContext.getRMApps()).thenReturn(new ConcurrentHashMap<ApplicationId, RMApp>());
    RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
    when(nlm.getResourceByLabel(any(), any())).thenReturn(res);
    when(rmContext.getNodeLabelManager()).thenReturn(nlm);
    // Queue "test" consumes 100% of the cluster, so its capacity and absolute
    // capacity are both 1.0f.
    Queue queue = createQueue("test", null, 1.0f, 1.0f);
    final String user = "user1";
    FiCaSchedulerApp app = new FiCaSchedulerApp(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    // Resource request
    Resource requestedResource = Resource.newInstance(1536, 2);
    app.getAppAttemptResourceUsage().incUsed(requestedResource);
    // In "test" queue, 1536 used is 15% of both the queue and the cluster
    assertEquals(15.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
    // Queue "test2" is a child of root and its capacity is 50% of root. As a
    // child of root, its absolute capaicty is also 50%.
    queue = createQueue("test2", null, 0.5f, 0.5f);
    app = new FiCaSchedulerApp(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    app.getAppAttemptResourceUsage().incUsed(requestedResource);
    // In "test2" queue, 1536 used is 30% of "test2" and 15% of the cluster.
    assertEquals(30.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
    // Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster.
    // Therefore, "test2.1" capacity is 50% and absolute capacity is 25%.
    AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f);
    app = new FiCaSchedulerApp(appAttId, user, qChild, qChild.getAbstractUsersManager(), rmContext);
    app.getAppAttemptResourceUsage().incUsed(requestedResource);
    // In "test2.1" queue, 1536 used is 60% of "test2.1" and 15% of the cluster.
    assertEquals(60.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
}