Examples with ResourceCalculator org.apache.hadoop.yarn.util.resource.ResourceCalculator used on opensource projects

Example 11 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestSchedulerUtils method testNormalizeRequestWithDominantResourceCalculator.

@Test(timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
    ResourceCalculator resourceCalculator = new DominantResourceCalculator();
    Resource minResource = Resources.createResource(1024, 1);
    Resource maxResource = Resources.createResource(10240, 10);
    Resource clusterResource = Resources.createResource(10 * 1024, 10);
    ResourceRequest ask = new ResourceRequestPBImpl();
    // case negative memory/vcores
    ask.setCapability(Resources.createResource(-1024, -1));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(minResource, ask.getCapability());
    // case zero memory/vcores
    ask.setCapability(Resources.createResource(0, 0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(minResource, ask.getCapability());
    assertEquals(1, ask.getCapability().getVirtualCores());
    assertEquals(1024, ask.getCapability().getMemorySize());
    // case non-zero memory & zero cores
    ask.setCapability(Resources.createResource(1536, 0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(Resources.createResource(2048, 1), ask.getCapability());
    assertEquals(1, ask.getCapability().getVirtualCores());
    assertEquals(2048, ask.getCapability().getMemorySize());
}

Example 12 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class SimpleCapacityReplanner method plan.

@Override
public void plan(Plan plan, List<ReservationDefinition> contracts) throws PlanningException {
    if (contracts != null) {
        throw new RuntimeException("SimpleCapacityReplanner cannot handle new reservation contracts");
    }
    ResourceCalculator resCalc = plan.getResourceCalculator();
    Resource totCap = plan.getTotalCapacity();
    long now = clock.getTime();
    // or the end of the planned sessions whichever comes first
    for (long t = now; (t < plan.getLastEndTime() && t < (now + lengthOfCheckZone)); t += plan.getStep()) {
        Resource excessCap = Resources.subtract(plan.getTotalCommittedResources(t), totCap);
        // if we are violating
        if (Resources.greaterThan(resCalc, totCap, excessCap, ZERO_RESOURCE)) {
            // sorted on reverse order of acceptance, so newest reservations first
            Set<ReservationAllocation> curReservations = new TreeSet<ReservationAllocation>(plan.getReservationsAtTime(t));
            for (Iterator<ReservationAllocation> resIter = curReservations.iterator(); resIter.hasNext() && Resources.greaterThan(resCalc, totCap, excessCap, ZERO_RESOURCE); ) {
                ReservationAllocation reservation = resIter.next();
                plan.deleteReservation(reservation.getReservationId());
                excessCap = Resources.subtract(excessCap, reservation.getResourcesAtTime(t));
                LOG.info("Removing reservation " + reservation.getReservationId() + " to repair physical-resource constraints in the plan: " + plan.getQueueName());
            }
        }
    }
}

Example 13 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class AbstractReservationSystem method loadPlan.

private void loadPlan(String planName, Map<ReservationId, ReservationAllocationStateProto> reservations) throws PlanningException {
    Plan plan = plans.get(planName);
    Resource minAllocation = getMinAllocation();
    ResourceCalculator rescCalculator = getResourceCalculator();
    for (Entry<ReservationId, ReservationAllocationStateProto> currentReservation : reservations.entrySet()) {
        plan.addReservation(ReservationSystemUtil.toInMemoryAllocation(planName, currentReservation.getKey(), currentReservation.getValue(), minAllocation, rescCalculator), true);
        resQMap.put(currentReservation.getKey(), planName);
    }
    LOG.info("Recovered reservations for Plan: {}", planName);
}

Example 14 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class AbstractReservationSystem method initializePlan.

protected Plan initializePlan(String planQueueName) throws YarnException {
    String planQueuePath = getPlanQueuePath(planQueueName);
    SharingPolicy adPolicy = getAdmissionPolicy(planQueuePath);
    adPolicy.init(planQueuePath, getReservationSchedulerConfiguration());
    // Calculate the max plan capacity
    Resource minAllocation = getMinAllocation();
    Resource maxAllocation = getMaxAllocation();
    ResourceCalculator rescCalc = getResourceCalculator();
    Resource totCap = getPlanQueueCapacity(planQueueName);
    Plan plan = new InMemoryPlan(getRootQueueMetrics(), adPolicy, getAgent(planQueuePath), totCap, planStepSize, rescCalc, minAllocation, maxAllocation, planQueueName, getReplanner(planQueuePath), getReservationSchedulerConfiguration().getMoveOnExpiry(planQueuePath), rmContext);
    LOG.info("Initialized plan {} based on reservable queue {}", plan.toString(), planQueueName);
    return plan;
}

Example 15 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class CapacityReservationSystem method getPlanQueueCapacity.

@Override
protected Resource getPlanQueueCapacity(String planQueueName) {
    Resource minAllocation = getMinAllocation();
    ResourceCalculator rescCalc = getResourceCalculator();
    CSQueue planQueue = capScheduler.getQueue(planQueueName);
    return rescCalc.multiplyAndNormalizeDown(capScheduler.getClusterResource(), planQueue.getAbsoluteCapacity(), minAllocation);
}