Examples with ResourceCalculator org.apache.hadoop.yarn.util.resource.ResourceCalculator used on opensource projects

Example 21 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testSteps.

@Test
public void testSteps() {
    ResourceCalculator resCalc = new DefaultResourceCalculator();
    RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc);
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    Set<Entry<ReservationInterval, Resource>> inputs = generateAllocation(start, alloc, true).entrySet();
    for (Entry<ReservationInterval, Resource> ip : inputs) {
        rleSparseVector.addInterval(ip.getKey(), ip.getValue());
    }
    LOG.info(rleSparseVector.toString());
    Assert.assertFalse(rleSparseVector.isEmpty());
    Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(99));
    Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + alloc.length + 1));
    for (int i = 0; i < alloc.length; i++) {
        Assert.assertEquals(Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rleSparseVector.getCapacityAtTime(start + i));
    }
    Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + alloc.length + 2));
    for (Entry<ReservationInterval, Resource> ip : inputs) {
        rleSparseVector.removeInterval(ip.getKey(), ip.getValue());
    }
    LOG.info(rleSparseVector.toString());
    for (int i = 0; i < alloc.length; i++) {
        Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(start + i));
    }
    Assert.assertTrue(rleSparseVector.isEmpty());
}

Example 22 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestSchedulerUtils method testNormalizeRequest.

@Test(timeout = 30000)
public void testNormalizeRequest() {
    ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
    final int minMemory = 1024;
    final int maxMemory = 8192;
    Resource minResource = Resources.createResource(minMemory, 0);
    Resource maxResource = Resources.createResource(maxMemory, 0);
    ResourceRequest ask = new ResourceRequestPBImpl();
    // case negative memory
    ask.setCapability(Resources.createResource(-1024));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(minMemory, ask.getCapability().getMemorySize());
    // case zero memory
    ask.setCapability(Resources.createResource(0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(minMemory, ask.getCapability().getMemorySize());
    // case memory is a multiple of minMemory
    ask.setCapability(Resources.createResource(2 * minMemory));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(2 * minMemory, ask.getCapability().getMemorySize());
    // case memory is not a multiple of minMemory
    ask.setCapability(Resources.createResource(minMemory + 10));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(2 * minMemory, ask.getCapability().getMemorySize());
    // case memory is equal to max allowed
    ask.setCapability(Resources.createResource(maxMemory));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(maxMemory, ask.getCapability().getMemorySize());
    // case memory is just less than max
    ask.setCapability(Resources.createResource(maxMemory - 10));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(maxMemory, ask.getCapability().getMemorySize());
    // max is not a multiple of min
    maxResource = Resources.createResource(maxMemory - 10, 0);
    ask.setCapability(Resources.createResource(maxMemory - 100));
    // multiple of minMemory > maxMemory, then reduce to maxMemory
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize());
    // ask is more than max
    maxResource = Resources.createResource(maxMemory, 0);
    ask.setCapability(Resources.createResource(maxMemory + 100));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize());
}

Example 23 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class FiCaSchedulerApp method getResourceUsageReport.

/**
   * Recalculates the per-app, percent of queue metric, specific to the
   * Capacity Scheduler.
   */
@Override
public ApplicationResourceUsageReport getResourceUsageReport() {
    try {
        // Use write lock here because
        // SchedulerApplicationAttempt#getResourceUsageReport updated fields
        // TODO: improve this
        writeLock.lock();
        ApplicationResourceUsageReport report = super.getResourceUsageReport();
        Resource cluster = rmContext.getScheduler().getClusterResource();
        Resource totalPartitionRes = rmContext.getNodeLabelManager().getResourceByLabel(getAppAMNodePartitionName(), cluster);
        ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
        if (!calc.isInvalidDivisor(totalPartitionRes)) {
            float queueAbsMaxCapPerPartition = ((AbstractCSQueue) getQueue()).getQueueCapacities().getAbsoluteCapacity(getAppAMNodePartitionName());
            float queueUsagePerc = calc.divide(totalPartitionRes, report.getUsedResources(), Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition)) * 100;
            report.setQueueUsagePercentage(queueUsagePerc);
        }
        return report;
    } finally {
        writeLock.unlock();
    }
}

Example 24 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class RMStateStoreTestBase method testReservationStateStore.

public void testReservationStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int start = 1;
    int[] alloc = { 10, 10, 10, 10, 10 };
    ResourceCalculator res = new DefaultResourceCalculator();
    Resource minAlloc = Resource.newInstance(1024, 1);
    boolean hasGang = true;
    String planName = "dedicated";
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    ReservationAllocation allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    assertAllocationStateEqual(allocation, allocationStateProto);
    // 1. Load empty store and verify no errors
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<String, Map<ReservationId, ReservationAllocationStateProto>> reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    // 2. Store single reservation and verify
    String reservationIdName = r1.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify new state
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 3. update state test
    alloc = new int[] { 6, 6, 6 };
    hasGang = false;
    allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 2, 2 + alloc.length, ReservationSystemTestUtil.generateAllocation(1L, 2L, alloc), res, minAlloc, hasGang);
    allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify updated reservation
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 4. add a second one and remove the first one
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    ReservationAllocation allocation2 = new InMemoryReservationAllocation(r2, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto2 = ReservationSystemUtil.buildStateProto(allocation2);
    String reservationIdName2 = r2.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto2, planName, reservationIdName2);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    // load state and verify r1 is removed and r2 is still there
    Map<ReservationId, ReservationAllocationStateProto> reservations;
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNotNull(reservations);
    ReservationAllocationStateProto storedReservationAllocation = reservations.get(r1);
    Assert.assertNull("Removed reservation should not be available in store", storedReservationAllocation);
    storedReservationAllocation = reservations.get(r2);
    assertAllocationStateEqual(allocationStateProto2, storedReservationAllocation);
    assertAllocationStateEqual(allocation2, storedReservationAllocation);
    // 5. remove last reservation removes the plan state
    rmContext.getStateStore().removeReservation(planName, reservationIdName2);
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNull(reservations);
}

Example 25 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method mockNested.

ParentQueue mockNested(Resource[] abs, int[] maxCap, Resource[] used, Resource[] pending, Resource[] reserved, int[] apps, Resource[] gran, int[] queues) {
    ResourceCalculator rc = mCS.getResourceCalculator();
    Resource tot = leafAbsCapacities(abs, queues);
    Deque<ParentQueue> pqs = new LinkedList<ParentQueue>();
    ParentQueue root = mockParentQueue(null, queues[0], pqs);
    ResourceUsage resUsage = new ResourceUsage();
    resUsage.setUsed(used[0]);
    resUsage.setReserved(reserved[0]);
    when(root.getQueueName()).thenReturn(CapacitySchedulerConfiguration.ROOT);
    when(root.getAbsoluteUsedCapacity()).thenReturn(Resources.divide(rc, tot, used[0], tot));
    when(root.getAbsoluteCapacity()).thenReturn(Resources.divide(rc, tot, abs[0], tot));
    when(root.getAbsoluteMaximumCapacity()).thenReturn(maxCap[0] / (float) tot.getMemorySize());
    when(root.getQueueResourceUsage()).thenReturn(resUsage);
    QueueCapacities rootQc = new QueueCapacities(true);
    rootQc.setAbsoluteUsedCapacity(Resources.divide(rc, tot, used[0], tot));
    rootQc.setAbsoluteCapacity(Resources.divide(rc, tot, abs[0], tot));
    rootQc.setAbsoluteMaximumCapacity(maxCap[0] / (float) tot.getMemorySize());
    when(root.getQueueCapacities()).thenReturn(rootQc);
    when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT);
    boolean preemptionDisabled = mockPreemptionStatus("root");
    when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
    for (int i = 1; i < queues.length; ++i) {
        final CSQueue q;
        final ParentQueue p = pqs.removeLast();
        final String queueName = "queue" + ((char) ('A' + i - 1));
        if (queues[i] > 0) {
            q = mockParentQueue(p, queues[i], pqs);
            ResourceUsage resUsagePerQueue = new ResourceUsage();
            resUsagePerQueue.setUsed(used[i]);
            resUsagePerQueue.setReserved(reserved[i]);
            when(q.getQueueResourceUsage()).thenReturn(resUsagePerQueue);
        } else {
            q = mockLeafQueue(p, tot, i, abs, used, pending, reserved, apps, gran);
        }
        when(q.getParent()).thenReturn(p);
        when(q.getQueueName()).thenReturn(queueName);
        when(q.getAbsoluteUsedCapacity()).thenReturn(Resources.divide(rc, tot, used[i], tot));
        when(q.getAbsoluteCapacity()).thenReturn(Resources.divide(rc, tot, abs[i], tot));
        when(q.getAbsoluteMaximumCapacity()).thenReturn(maxCap[i] / (float) tot.getMemorySize());
        // We need to make these fields to QueueCapacities
        QueueCapacities qc = new QueueCapacities(false);
        qc.setAbsoluteUsedCapacity(Resources.divide(rc, tot, used[i], tot));
        qc.setAbsoluteCapacity(Resources.divide(rc, tot, abs[i], tot));
        qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize());
        when(q.getQueueCapacities()).thenReturn(qc);
        String parentPathName = p.getQueuePath();
        parentPathName = (parentPathName == null) ? "root" : parentPathName;
        String queuePathName = (parentPathName + "." + queueName).replace("/", "root");
        when(q.getQueuePath()).thenReturn(queuePathName);
        preemptionDisabled = mockPreemptionStatus(queuePathName);
        when(q.getPreemptionDisabled()).thenReturn(preemptionDisabled);
    }
    assert 0 == pqs.size();
    return root;
}