Examples with DefaultResourceCalculator org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator used on opensource projects

Example 21 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testMergeSubtract.

@Test
public void testMergeSubtract() throws PlanningException {
    TreeMap<Long, Resource> a = new TreeMap<>();
    TreeMap<Long, Resource> b = new TreeMap<>();
    setupArrays(a, b);
    RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
    RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
    RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.subtract, 0, 60);
    System.out.println(out);
    long[] time = { 10, 11, 20, 22, 30, 33, 43, 50, 60 };
    int[] alloc = { 5, 0, 5, 0, 5, 0, 10, -10 };
    validate(out, time, alloc);
}

Example 22 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testMergeSpeed.

@Test
@Ignore
public void testMergeSpeed() throws PlanningException {
    for (int j = 0; j < 100; j++) {
        TreeMap<Long, Resource> a = new TreeMap<>();
        TreeMap<Long, Resource> b = new TreeMap<>();
        Random rand = new Random();
        long startA = 0;
        long startB = 0;
        for (int i = 0; i < 1000 + rand.nextInt(9000); i++) {
            startA += rand.nextInt(100);
            startB += rand.nextInt(100);
            a.put(startA, Resource.newInstance(rand.nextInt(10240), rand.nextInt(10)));
            b.put(startB, Resource.newInstance(rand.nextInt(10240), rand.nextInt(10)));
        }
        RLESparseResourceAllocation rleA = new RLESparseResourceAllocation(a, new DefaultResourceCalculator());
        RLESparseResourceAllocation rleB = new RLESparseResourceAllocation(b, new DefaultResourceCalculator());
        long start = System.currentTimeMillis();
        RLESparseResourceAllocation out = RLESparseResourceAllocation.merge(new DefaultResourceCalculator(), Resource.newInstance(100 * 128 * 1024, 100 * 32), rleA, rleB, RLEOperator.add, Long.MIN_VALUE, Long.MAX_VALUE);
        long end = System.currentTimeMillis();
        System.out.println(" Took: " + (end - start) + "ms ");
    }
}

Example 23 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testSteps.

@Test
public void testSteps() {
    ResourceCalculator resCalc = new DefaultResourceCalculator();
    RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc);
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    Set<Entry<ReservationInterval, Resource>> inputs = generateAllocation(start, alloc, true).entrySet();
    for (Entry<ReservationInterval, Resource> ip : inputs) {
        rleSparseVector.addInterval(ip.getKey(), ip.getValue());
    }
    LOG.info(rleSparseVector.toString());
    Assert.assertFalse(rleSparseVector.isEmpty());
    Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(99));
    Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + alloc.length + 1));
    for (int i = 0; i < alloc.length; i++) {
        Assert.assertEquals(Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rleSparseVector.getCapacityAtTime(start + i));
    }
    Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + alloc.length + 2));
    for (Entry<ReservationInterval, Resource> ip : inputs) {
        rleSparseVector.removeInterval(ip.getKey(), ip.getValue());
    }
    LOG.info(rleSparseVector.toString());
    for (int i = 0; i < alloc.length; i++) {
        Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + i));
    }
    Assert.assertTrue(rleSparseVector.isEmpty());
}

Example 24 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestSchedulerUtils method testNormalizeRequest.

@Test(timeout = 30000)
public void testNormalizeRequest() {
    ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
    final int minMemory = 1024;
    final int maxMemory = 8192;
    Resource minResource = Resources.createResource(minMemory, 0);
    Resource maxResource = Resources.createResource(maxMemory, 0);
    ResourceRequest ask = new ResourceRequestPBImpl();
    // case negative memory
    ask.setCapability(Resources.createResource(-1024));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(minMemory, ask.getCapability().getMemorySize());
    // case zero memory
    ask.setCapability(Resources.createResource(0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(minMemory, ask.getCapability().getMemorySize());
    // case memory is a multiple of minMemory
    ask.setCapability(Resources.createResource(2 * minMemory));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(2 * minMemory, ask.getCapability().getMemorySize());
    // case memory is not a multiple of minMemory
    ask.setCapability(Resources.createResource(minMemory + 10));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(2 * minMemory, ask.getCapability().getMemorySize());
    // case memory is equal to max allowed
    ask.setCapability(Resources.createResource(maxMemory));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(maxMemory, ask.getCapability().getMemorySize());
    // case memory is just less than max
    ask.setCapability(Resources.createResource(maxMemory - 10));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(maxMemory, ask.getCapability().getMemorySize());
    // max is not a multiple of min
    maxResource = Resources.createResource(maxMemory - 10, 0);
    ask.setCapability(Resources.createResource(maxMemory - 100));
    // multiple of minMemory > maxMemory, then reduce to maxMemory
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize());
    // ask is more than max
    maxResource = Resources.createResource(maxMemory, 0);
    ask.setCapability(Resources.createResource(maxMemory + 100));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize());
}

Example 25 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestSchedulerApplicationAttempt method testAppPercentages.

@Test
public void testAppPercentages() throws Exception {
    FifoScheduler scheduler = mock(FifoScheduler.class);
    when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(10 * 1024, 10));
    when(scheduler.getResourceCalculator()).thenReturn(new DefaultResourceCalculator());
    ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getEpoch()).thenReturn(3L);
    when(rmContext.getScheduler()).thenReturn(scheduler);
    final String user = "user1";
    Queue queue = createQueue("test", null);
    SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    // Resource request
    Resource requestedResource = Resource.newInstance(1536, 2);
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(15.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
    queue = createQueue("test2", null, 0.5f);
    app = new SchedulerApplicationAttempt(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(30.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
    app.attemptResourceUsage.incUsed(requestedResource);
    app.attemptResourceUsage.incUsed(requestedResource);
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(120.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(60.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
    queue = createQueue("test3", null, 0.0f);
    app = new SchedulerApplicationAttempt(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    // Resource request
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(0.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
}