Examples with ResourceCalculator org.apache.hadoop.yarn.util.resource.ResourceCalculator used on opensource projects

Example 6 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class AbstractReservationSystem method initializePlan.

protected Plan initializePlan(String planQueueName) throws YarnException {
    String planQueuePath = getPlanQueuePath(planQueueName);
    SharingPolicy adPolicy = getAdmissionPolicy(planQueuePath);
    adPolicy.init(planQueuePath, getReservationSchedulerConfiguration());
    // Calculate the max plan capacity
    Resource minAllocation = getMinAllocation();
    Resource maxAllocation = getMaxAllocation();
    ResourceCalculator rescCalc = getResourceCalculator();
    Resource totCap = getPlanQueueCapacity(planQueueName);
    Plan plan = new InMemoryPlan(getRootQueueMetrics(), adPolicy, getAgent(planQueuePath), totCap, planStepSize, rescCalc, minAllocation, maxAllocation, planQueueName, getReplanner(planQueuePath), getReservationSchedulerConfiguration().getMoveOnExpiry(planQueuePath), rmContext);
    LOG.info("Initialized plan {} based on reservable queue {}", plan.toString(), planQueueName);
    return plan;
}

Example 7 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class CapacityReservationSystem method getPlanQueueCapacity.

@Override
protected Resource getPlanQueueCapacity(String planQueueName) {
    Resource minAllocation = getMinAllocation();
    ResourceCalculator rescCalc = getResourceCalculator();
    CSQueue planQueue = capScheduler.getQueue(planQueueName);
    return rescCalc.multiplyAndNormalizeDown(capScheduler.getClusterResource(), planQueue.getAbsoluteCapacity(), minAllocation);
}

Example 8 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method mockLeafQueue.

@SuppressWarnings("rawtypes")
LeafQueue mockLeafQueue(ParentQueue p, Resource tot, int i, Resource[] abs, Resource[] used, Resource[] pending, Resource[] reserved, int[] apps, Resource[] gran) {
    LeafQueue lq = mock(LeafQueue.class);
    ResourceCalculator rc = mCS.getResourceCalculator();
    List<ApplicationAttemptId> appAttemptIdList = new ArrayList<ApplicationAttemptId>();
    when(lq.getTotalPendingResourcesConsideringUserLimit(isA(Resource.class), isA(String.class), eq(false))).thenReturn(pending[i]);
    when(lq.getTotalPendingResourcesConsideringUserLimit(isA(Resource.class), isA(String.class), eq(true))).thenReturn(Resources.componentwiseMax(Resources.subtract(pending[i], reserved[i] == null ? Resources.none() : reserved[i]), Resources.none()));
    // need to set pending resource in resource usage as well
    ResourceUsage ru = new ResourceUsage();
    ru.setPending(pending[i]);
    ru.setUsed(used[i]);
    ru.setReserved(reserved[i]);
    when(lq.getQueueResourceUsage()).thenReturn(ru);
    // consider moving where CapacityScheduler::comparator accessible
    final NavigableSet<FiCaSchedulerApp> qApps = new TreeSet<FiCaSchedulerApp>(new Comparator<FiCaSchedulerApp>() {

        @Override
        public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) {
            return a1.getApplicationAttemptId().compareTo(a2.getApplicationAttemptId());
        }
    });
    // applications are added in global L->R order in queues
    if (apps[i] != 0) {
        Resource aUsed = Resources.divideAndCeil(rc, used[i], apps[i]);
        Resource aPending = Resources.divideAndCeil(rc, pending[i], apps[i]);
        Resource aReserve = Resources.divideAndCeil(rc, reserved[i], apps[i]);
        for (int a = 0; a < apps[i]; ++a) {
            FiCaSchedulerApp mockFiCaApp = mockApp(i, appAlloc, aUsed, aPending, aReserve, gran[i]);
            qApps.add(mockFiCaApp);
            ++appAlloc;
            appAttemptIdList.add(mockFiCaApp.getApplicationAttemptId());
        }
        when(mCS.getAppsInQueue("queue" + (char) ('A' + i - 1))).thenReturn(appAttemptIdList);
    }
    when(lq.getApplications()).thenReturn(qApps);
    @SuppressWarnings("unchecked") OrderingPolicy<FiCaSchedulerApp> so = mock(OrderingPolicy.class);
    when(so.getPreemptionIterator()).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            return qApps.descendingIterator();
        }
    });
    when(lq.getOrderingPolicy()).thenReturn(so);
    if (setAMResourcePercent != 0.0f) {
        when(lq.getMaxAMResourcePerQueuePercent()).thenReturn(setAMResourcePercent);
    }
    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
    when(lq.getReadLock()).thenReturn(lock.readLock());
    when(lq.getPriority()).thenReturn(Priority.newInstance(0));
    p.getChildQueues().add(lq);
    return lq;
}

Example 9 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method mockApp.

FiCaSchedulerApp mockApp(int qid, int id, Resource used, Resource pending, Resource reserved, Resource gran) {
    FiCaSchedulerApp app = mock(FiCaSchedulerApp.class);
    ResourceCalculator rc = mCS.getResourceCalculator();
    ApplicationId appId = ApplicationId.newInstance(TS, id);
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(appId, 0);
    when(app.getApplicationId()).thenReturn(appId);
    when(app.getApplicationAttemptId()).thenReturn(appAttId);
    int cAlloc = 0;
    Resource unit = gran;
    List<RMContainer> cReserved = new ArrayList<RMContainer>();
    Resource resIter = Resource.newInstance(0, 0);
    for (; Resources.lessThan(rc, clusterResources, resIter, reserved); Resources.addTo(resIter, gran)) {
        cReserved.add(mockContainer(appAttId, cAlloc, unit, priority.CONTAINER.getValue()));
        ++cAlloc;
    }
    when(app.getReservedContainers()).thenReturn(cReserved);
    List<RMContainer> cLive = new ArrayList<RMContainer>();
    Resource usedIter = Resource.newInstance(0, 0);
    int i = 0;
    for (; Resources.lessThan(rc, clusterResources, usedIter, used); Resources.addTo(usedIter, gran)) {
        if (setAMContainer && i == 0) {
            cLive.add(mockContainer(appAttId, cAlloc, unit, priority.AMCONTAINER.getValue()));
        } else if (setLabeledContainer && i == 1) {
            cLive.add(mockContainer(appAttId, cAlloc, unit, priority.LABELEDCONTAINER.getValue()));
            Resources.addTo(used, Resource.newInstance(1, 1));
        } else {
            cLive.add(mockContainer(appAttId, cAlloc, unit, priority.CONTAINER.getValue()));
        }
        ++cAlloc;
        ++i;
    }
    when(app.getLiveContainers()).thenReturn(cLive);
    return app;
}

Example 10 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestRLESparseResourceAllocation method testZeroAllocation.

@Test
public void testZeroAllocation() {
    ResourceCalculator resCalc = new DefaultResourceCalculator();
    RLESparseResourceAllocation rleSparseVector = new RLESparseResourceAllocation(resCalc);
    rleSparseVector.addInterval(new ReservationInterval(0, Long.MAX_VALUE), Resource.newInstance(0, 0));
    LOG.info(rleSparseVector.toString());
    Assert.assertEquals(Resource.newInstance(0, 0), rleSparseVector.getCapacityAtTime(new Random().nextLong()));
    Assert.assertTrue(rleSparseVector.isEmpty());
}