Examples with ResourceCalculator org.apache.hadoop.yarn.util.resource.ResourceCalculator used on opensource projects

Example 1 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestClientRMService method mockYarnScheduler.

private static YarnScheduler mockYarnScheduler() throws YarnException {
    YarnScheduler yarnScheduler = mock(YarnScheduler.class);
    when(yarnScheduler.getMinimumResourceCapability()).thenReturn(Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
    when(yarnScheduler.getMaximumResourceCapability()).thenReturn(Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
    when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn(Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102)));
    when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(Arrays.asList(getApplicationAttemptId(103)));
    ApplicationAttemptId attemptId = getApplicationAttemptId(1);
    when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
    ResourceCalculator rs = mock(ResourceCalculator.class);
    when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
    when(yarnScheduler.checkAndGetApplicationPriority(any(Priority.class), any(UserGroupInformation.class), anyString(), any(ApplicationId.class))).thenReturn(Priority.newInstance(0));
    return yarnScheduler;
}

Example 2 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class TestAppManager method mockResourceScheduler.

private static ResourceScheduler mockResourceScheduler() {
    ResourceScheduler scheduler = mock(ResourceScheduler.class);
    when(scheduler.getMinimumResourceCapability()).thenReturn(Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
    when(scheduler.getMaximumResourceCapability()).thenReturn(Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
    ResourceCalculator rs = mock(ResourceCalculator.class);
    when(scheduler.getResourceCalculator()).thenReturn(rs);
    return scheduler;
}

Example 3 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class SchedulerApplicationAttempt method getResourceUsageReport.

public ApplicationResourceUsageReport getResourceUsageReport() {
    try {
        writeLock.lock();
        AggregateAppResourceUsage runningResourceUsage = getRunningAggregateAppResourceUsage();
        Resource usedResourceClone = Resources.clone(attemptResourceUsage.getAllUsed());
        Resource reservedResourceClone = Resources.clone(attemptResourceUsage.getReserved());
        Resource cluster = rmContext.getScheduler().getClusterResource();
        ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
        float queueUsagePerc = 0.0f;
        float clusterUsagePerc = 0.0f;
        if (!calc.isInvalidDivisor(cluster)) {
            float queueCapacityPerc = queue.getQueueInfo(false, false).getCapacity();
            if (queueCapacityPerc != 0) {
                queueUsagePerc = calc.divide(cluster, usedResourceClone, Resources.multiply(cluster, queueCapacityPerc)) * 100;
            }
            clusterUsagePerc = calc.divide(cluster, usedResourceClone, cluster) * 100;
        }
        return ApplicationResourceUsageReport.newInstance(liveContainers.size(), reservedContainers.size(), usedResourceClone, reservedResourceClone, Resources.add(usedResourceClone, reservedResourceClone), runningResourceUsage.getMemorySeconds(), runningResourceUsage.getVcoreSeconds(), queueUsagePerc, clusterUsagePerc, 0, 0);
    } finally {
        writeLock.unlock();
    }
}

Example 4 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class SimpleCapacityReplanner method plan.

@Override
public void plan(Plan plan, List<ReservationDefinition> contracts) throws PlanningException {
    if (contracts != null) {
        throw new RuntimeException("SimpleCapacityReplanner cannot handle new reservation contracts");
    }
    ResourceCalculator resCalc = plan.getResourceCalculator();
    Resource totCap = plan.getTotalCapacity();
    long now = clock.getTime();
    // or the end of the planned sessions whichever comes first
    for (long t = now; (t < plan.getLastEndTime() && t < (now + lengthOfCheckZone)); t += plan.getStep()) {
        Resource excessCap = Resources.subtract(plan.getTotalCommittedResources(t), totCap);
        // if we are violating
        if (Resources.greaterThan(resCalc, totCap, excessCap, ZERO_RESOURCE)) {
            // sorted on reverse order of acceptance, so newest reservations first
            Set<ReservationAllocation> curReservations = new TreeSet<ReservationAllocation>(plan.getReservationsAtTime(t));
            for (Iterator<ReservationAllocation> resIter = curReservations.iterator(); resIter.hasNext() && Resources.greaterThan(resCalc, totCap, excessCap, ZERO_RESOURCE); ) {
                ReservationAllocation reservation = resIter.next();
                plan.deleteReservation(reservation.getReservationId());
                excessCap = Resources.subtract(excessCap, reservation.getResourcesAtTime(t));
                LOG.info("Removing reservation " + reservation.getReservationId() + " to repair physical-resource constraints in the plan: " + plan.getQueueName());
            }
        }
    }
}

Example 5 with ResourceCalculator

use of org.apache.hadoop.yarn.util.resource.ResourceCalculator in project hadoop by apache.

the class AbstractReservationSystem method loadPlan.

private void loadPlan(String planName, Map<ReservationId, ReservationAllocationStateProto> reservations) throws PlanningException {
    Plan plan = plans.get(planName);
    Resource minAllocation = getMinAllocation();
    ResourceCalculator rescCalculator = getResourceCalculator();
    for (Entry<ReservationId, ReservationAllocationStateProto> currentReservation : reservations.entrySet()) {
        plan.addReservation(ReservationSystemUtil.toInMemoryAllocation(planName, currentReservation.getKey(), currentReservation.getValue(), minAllocation, rescCalculator), true);
        resQMap.put(currentReservation.getKey(), planName);
    }
    LOG.info("Recovered reservations for Plan: {}", planName);
}