Examples with TreeMap java.util.TreeMap used on opensource projects

Example 71 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class CommonNodeLabelsManager method normalizeNodeIdToLabels.

protected Map<NodeId, Set<String>> normalizeNodeIdToLabels(Map<NodeId, Set<String>> nodeIdToLabels) {
    Map<NodeId, Set<String>> newMap = new TreeMap<NodeId, Set<String>>();
    for (Entry<NodeId, Set<String>> entry : nodeIdToLabels.entrySet()) {
        NodeId id = entry.getKey();
        Set<String> labels = entry.getValue();
        newMap.put(id, normalizeLabels(labels));
    }
    return newMap;
}

Example 72 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class CapacityOverTimePolicy method validate.

/**
   * The validation algorithm walks over the RLE encoded allocation and
   * checks that for all transition points (when the start or end of the
   * checking window encounters a value in the RLE). At this point it
   * checkes whether the integral computed exceeds the quota limit. Note that
   * this might not find the exact time of a violation, but if a violation
   * exists it will find it. The advantage is a much lower number of checks
   * as compared to time-slot by time-slot checks.
   *
   * @param plan the plan to validate against
   * @param reservation the reservation allocation to test.
   * @throws PlanningException if the validation fails.
   */
@Override
public void validate(Plan plan, ReservationAllocation reservation) throws PlanningException {
    // cluster limits, and 3) maxInst (via override of available)
    try {
        super.validate(plan, reservation);
    } catch (PlanningException p) {
        //wrap it in proper quota exception
        throw new PlanningQuotaException(p);
    }
    //---- check for integral violations of capacity --------
    // Gather a view of what to check (curr allocation of user, minus old
    // version of this reservation, plus new version)
    RLESparseResourceAllocation consumptionForUserOverTime = plan.getConsumptionForUserOverTime(reservation.getUser(), reservation.getStartTime() - validWindow, reservation.getEndTime() + validWindow);
    ReservationAllocation old = plan.getReservationById(reservation.getReservationId());
    if (old != null) {
        consumptionForUserOverTime = RLESparseResourceAllocation.merge(plan.getResourceCalculator(), plan.getTotalCapacity(), consumptionForUserOverTime, old.getResourcesOverTime(), RLEOperator.add, reservation.getStartTime() - validWindow, reservation.getEndTime() + validWindow);
    }
    RLESparseResourceAllocation resRLE = reservation.getResourcesOverTime();
    RLESparseResourceAllocation toCheck = RLESparseResourceAllocation.merge(plan.getResourceCalculator(), plan.getTotalCapacity(), consumptionForUserOverTime, resRLE, RLEOperator.add, Long.MIN_VALUE, Long.MAX_VALUE);
    NavigableMap<Long, Resource> integralUp = new TreeMap<>();
    NavigableMap<Long, Resource> integralDown = new TreeMap<>();
    long prevTime = toCheck.getEarliestStartTime();
    IntegralResource prevResource = new IntegralResource(0L, 0L);
    IntegralResource runningTot = new IntegralResource(0L, 0L);
    // add intermediate points
    Map<Long, Resource> temp = new TreeMap<>();
    for (Map.Entry<Long, Resource> pointToCheck : toCheck.getCumulative().entrySet()) {
        Long timeToCheck = pointToCheck.getKey();
        Resource resourceToCheck = pointToCheck.getValue();
        Long nextPoint = toCheck.getCumulative().higherKey(timeToCheck);
        if (nextPoint == null || toCheck.getCumulative().get(nextPoint) == null) {
            continue;
        }
        for (int i = 1; i <= (nextPoint - timeToCheck) / validWindow; i++) {
            temp.put(timeToCheck + (i * validWindow), resourceToCheck);
        }
    }
    temp.putAll(toCheck.getCumulative());
    // compute point-wise integral for the up-fronts and down-fronts
    for (Map.Entry<Long, Resource> currPoint : temp.entrySet()) {
        Long currTime = currPoint.getKey();
        Resource currResource = currPoint.getValue();
        //add to running total current contribution
        prevResource.multiplyBy(currTime - prevTime);
        runningTot.add(prevResource);
        integralUp.put(currTime, normalizeToResource(runningTot, validWindow));
        integralDown.put(currTime + validWindow, normalizeToResource(runningTot, validWindow));
        if (currResource != null) {
            prevResource.memory = currResource.getMemorySize();
            prevResource.vcores = currResource.getVirtualCores();
        } else {
            prevResource.memory = 0L;
            prevResource.vcores = 0L;
        }
        prevTime = currTime;
    }
    // compute final integral as delta of up minus down transitions
    RLESparseResourceAllocation intUp = new RLESparseResourceAllocation(integralUp, plan.getResourceCalculator());
    RLESparseResourceAllocation intDown = new RLESparseResourceAllocation(integralDown, plan.getResourceCalculator());
    RLESparseResourceAllocation integral = RLESparseResourceAllocation.merge(plan.getResourceCalculator(), plan.getTotalCapacity(), intUp, intDown, RLEOperator.subtract, Long.MIN_VALUE, Long.MAX_VALUE);
    // define over-time integral limit
    // note: this is aligned with the normalization done above
    NavigableMap<Long, Resource> tlimit = new TreeMap<>();
    Resource maxAvgRes = Resources.multiply(plan.getTotalCapacity(), maxAvg);
    tlimit.put(toCheck.getEarliestStartTime() - validWindow, maxAvgRes);
    RLESparseResourceAllocation targetLimit = new RLESparseResourceAllocation(tlimit, plan.getResourceCalculator());
    // compare using merge() limit with integral
    try {
        RLESparseResourceAllocation.merge(plan.getResourceCalculator(), plan.getTotalCapacity(), targetLimit, integral, RLEOperator.subtractTestNonNegative, reservation.getStartTime() - validWindow, reservation.getEndTime() + validWindow);
    } catch (PlanningException p) {
        throw new PlanningQuotaException("Integral (avg over time) quota capacity " + maxAvg + " over a window of " + validWindow / 1000 + " seconds, " + " would be exceeded by accepting reservation: " + reservation.getReservationId(), p);
    }
}

Example 73 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class CapacityOverTimePolicy method availableResources.

@Override
public RLESparseResourceAllocation availableResources(RLESparseResourceAllocation available, Plan plan, String user, ReservationId oldId, long start, long end) throws PlanningException {
    // this only propagates the instantaneous maxInst properties, while
    // the time-varying one depends on the current allocation as well
    // and are not easily captured here
    Resource planTotalCapacity = plan.getTotalCapacity();
    Resource maxInsRes = Resources.multiply(planTotalCapacity, maxInst);
    NavigableMap<Long, Resource> instQuota = new TreeMap<Long, Resource>();
    instQuota.put(start, maxInsRes);
    RLESparseResourceAllocation instRLEQuota = new RLESparseResourceAllocation(instQuota, plan.getResourceCalculator());
    RLESparseResourceAllocation used = plan.getConsumptionForUserOverTime(user, start, end);
    // add back in old reservation used resources if any
    ReservationAllocation old = plan.getReservationById(oldId);
    if (old != null) {
        used = RLESparseResourceAllocation.merge(plan.getResourceCalculator(), Resources.clone(plan.getTotalCapacity()), used, old.getResourcesOverTime(), RLEOperator.subtract, start, end);
    }
    instRLEQuota = RLESparseResourceAllocation.merge(plan.getResourceCalculator(), planTotalCapacity, instRLEQuota, used, RLEOperator.subtract, start, end);
    instRLEQuota = RLESparseResourceAllocation.merge(plan.getResourceCalculator(), planTotalCapacity, available, instRLEQuota, RLEOperator.min, start, end);
    return instRLEQuota;
}

Example 74 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class RLESparseResourceAllocation method merge.

private static NavigableMap<Long, Resource> merge(ResourceCalculator resCalc, Resource clusterResource, NavigableMap<Long, Resource> a, NavigableMap<Long, Resource> b, long start, long end, RLEOperator operator) throws PlanningException {
    // handle special cases of empty input
    if (a == null || a.isEmpty()) {
        if (operator == RLEOperator.subtract || operator == RLEOperator.subtractTestNonNegative) {
            return negate(operator, b);
        } else {
            return b;
        }
    }
    if (b == null || b.isEmpty()) {
        return a;
    }
    // define iterators and support variables
    Iterator<Entry<Long, Resource>> aIt = a.entrySet().iterator();
    Iterator<Entry<Long, Resource>> bIt = b.entrySet().iterator();
    Entry<Long, Resource> curA = aIt.next();
    Entry<Long, Resource> curB = bIt.next();
    Entry<Long, Resource> lastA = null;
    Entry<Long, Resource> lastB = null;
    boolean aIsDone = false;
    boolean bIsDone = false;
    TreeMap<Long, Resource> out = new TreeMap<Long, Resource>();
    while (!(curA.equals(lastA) && curB.equals(lastB))) {
        Resource outRes;
        long time = -1;
        // curA is smaller than curB
        if (bIsDone || (curA.getKey() < curB.getKey() && !aIsDone)) {
            outRes = combineValue(operator, resCalc, clusterResource, curA, lastB);
            time = (curA.getKey() < start) ? start : curA.getKey();
            lastA = curA;
            if (aIt.hasNext()) {
                curA = aIt.next();
            } else {
                aIsDone = true;
            }
        } else {
            // curB is smaller than curA
            if (aIsDone || (curA.getKey() > curB.getKey() && !bIsDone)) {
                outRes = combineValue(operator, resCalc, clusterResource, lastA, curB);
                time = (curB.getKey() < start) ? start : curB.getKey();
                lastB = curB;
                if (bIt.hasNext()) {
                    curB = bIt.next();
                } else {
                    bIsDone = true;
                }
            } else {
                // curA is equal to curB
                outRes = combineValue(operator, resCalc, clusterResource, curA, curB);
                time = (curA.getKey() < start) ? start : curA.getKey();
                lastA = curA;
                if (aIt.hasNext()) {
                    curA = aIt.next();
                } else {
                    aIsDone = true;
                }
                lastB = curB;
                if (bIt.hasNext()) {
                    curB = bIt.next();
                } else {
                    bIsDone = true;
                }
            }
        }
        // add to out if not redundant
        addIfNeeded(out, time, outRes);
    }
    addIfNeeded(out, end, null);
    return out;
}

Example 75 with TreeMap

use of java.util.TreeMap in project hadoop by apache.

the class RLESparseResourceAllocation method toIntervalMap.

/**
   * Returns the representation of the current resources allocated over time as
   * an interval map (in the defined non-null range).
   *
   * @return the representation of the current resources allocated over time as
   *         an interval map.
   */
public Map<ReservationInterval, Resource> toIntervalMap() {
    readLock.lock();
    try {
        Map<ReservationInterval, Resource> allocations = new TreeMap<ReservationInterval, Resource>();
        // Empty
        if (isEmpty()) {
            return allocations;
        }
        Map.Entry<Long, Resource> lastEntry = null;
        for (Map.Entry<Long, Resource> entry : cumulativeCapacity.entrySet()) {
            if (lastEntry != null && entry.getValue() != null) {
                ReservationInterval interval = new ReservationInterval(lastEntry.getKey(), entry.getKey());
                Resource resource = lastEntry.getValue();
                allocations.put(interval, resource);
            }
            lastEntry = entry;
        }
        return allocations;
    } finally {
        readLock.unlock();
    }
}