Examples with ReservationInterval org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval used on opensource projects

Example 6 with ReservationInterval

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval in project hadoop by apache.

the class IterativePlanner method computeJobAllocation.

@Override
public RLESparseResourceAllocation computeJobAllocation(Plan plan, ReservationId reservationId, ReservationDefinition reservation, String user) throws PlanningException {
    // Initialize
    initialize(plan, reservationId, reservation);
    // Create the allocations data structure
    RLESparseResourceAllocation allocations = new RLESparseResourceAllocation(plan.getResourceCalculator());
    StageProvider stageProvider = new StageProvider(allocateLeft, reservation);
    // Current stage
    ReservationRequest currentReservationStage;
    // Stage deadlines
    long stageDeadline = stepRoundDown(reservation.getDeadline(), step);
    long successorStartingTime = -1;
    long predecessorEndTime = stepRoundDown(reservation.getArrival(), step);
    long stageArrivalTime = -1;
    // Iterate the stages in reverse order
    while (stageProvider.hasNext()) {
        // Get current stage
        currentReservationStage = stageProvider.next();
        // Validate that the ReservationRequest respects basic constraints
        validateInputStage(plan, currentReservationStage);
        if (allocateLeft) {
            stageArrivalTime = predecessorEndTime;
        } else {
            stageArrivalTime = reservation.getArrival();
            if (jobType == ReservationRequestInterpreter.R_ORDER || jobType == ReservationRequestInterpreter.R_ORDER_NO_GAP) {
                stageArrivalTime = computeEarliestStartingTime(plan, reservation, stageProvider.getCurrentIndex(), currentReservationStage, stageDeadline);
            }
            stageArrivalTime = stepRoundUp(stageArrivalTime, step);
            stageArrivalTime = Math.max(stageArrivalTime, reservation.getArrival());
        }
        // Compute the allocation of a single stage
        Map<ReservationInterval, Resource> curAlloc = computeStageAllocation(plan, currentReservationStage, stageArrivalTime, stageDeadline, user, reservationId);
        // (unless it's an ANY job, then we simply continue).
        if (curAlloc == null) {
            // If it's an ANY job, we can move to the next possible request
            if (jobType == ReservationRequestInterpreter.R_ANY) {
                continue;
            }
            // Otherwise, the job cannot be allocated
            throw new PlanningException("The request cannot be satisfied");
        }
        // Get the start & end time of the current allocation
        Long stageStartTime = findEarliestTime(curAlloc);
        Long stageEndTime = findLatestTime(curAlloc);
        // If we did find an allocation for the stage, add it
        for (Entry<ReservationInterval, Resource> entry : curAlloc.entrySet()) {
            allocations.addInterval(entry.getKey(), entry.getValue());
        }
        // If this is an ANY clause, we have finished
        if (jobType == ReservationRequestInterpreter.R_ANY) {
            break;
        }
        // If ORDER job, set the stageDeadline of the next stage to be processed
        if (jobType == ReservationRequestInterpreter.R_ORDER || jobType == ReservationRequestInterpreter.R_ORDER_NO_GAP) {
            // note that the test is different left-to-right and right-to-left
            if (jobType == ReservationRequestInterpreter.R_ORDER_NO_GAP && successorStartingTime != -1 && ((allocateLeft && predecessorEndTime < stageStartTime) || (!allocateLeft && (stageEndTime < successorStartingTime))) || (!isNonPreemptiveAllocation(curAlloc))) {
                throw new PlanningException("The allocation found does not respect ORDER_NO_GAP");
            }
            if (allocateLeft) {
                // Store the stageStartTime and set the new stageDeadline
                predecessorEndTime = stageEndTime;
            } else {
                // Store the stageStartTime and set the new stageDeadline
                successorStartingTime = stageStartTime;
                stageDeadline = stageStartTime;
            }
        }
    }
    // If the allocation is empty, return an error
    if (allocations.isEmpty()) {
        throw new PlanningException("The request cannot be satisfied");
    }
    return allocations;
}

Example 7 with ReservationInterval

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval in project hadoop by apache.

the class TestGreedyReservationAgent method prepareBasicPlan.

private void prepareBasicPlan() throws PlanningException {
    // insert in the reservation a couple of controlled reservations, to create
    // conditions for assignment that are non-empty
    int[] f = { 10, 10, 20, 20, 20, 10, 10 };
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(0, 0 + f.length * step, f.length * step);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(ReservationSystemTestUtil.getNewReservationId(), rDef, "u1", "dedicated", 0L, 0L + f.length * step, ReservationSystemTestUtil.generateAllocation(0, step, f), res, minAlloc), false));
    int[] f2 = { 5, 5, 5, 5, 5, 5, 5 };
    Map<ReservationInterval, Resource> alloc = ReservationSystemTestUtil.generateAllocation(5000, step, f2);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(ReservationSystemTestUtil.getNewReservationId(), rDef, "u1", "dedicated", 5000, 5000 + f2.length * step, alloc, res, minAlloc), false));
    System.out.println("--------BEFORE AGENT----------");
    System.out.println(plan.toString());
    System.out.println(plan.toCumulativeString());
}

Example 8 with ReservationInterval

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval in project hadoop by apache.

the class PlanningAlgorithm method allocateUser.

/**
   * Performs the actual allocation for a ReservationDefinition within a Plan.
   *
   * @param reservationId the identifier of the reservation
   * @param user the user who owns the reservation
   * @param plan the Plan to which the reservation must be fitted
   * @param contract encapsulates the resources required by the user for his
   *          session
   * @param oldReservation the existing reservation (null if none)
   * @return whether the allocateUser function was successful or not
   *
   * @throws PlanningException if the session cannot be fitted into the plan
   * @throws ContractValidationException
   */
protected boolean allocateUser(ReservationId reservationId, String user, Plan plan, ReservationDefinition contract, ReservationAllocation oldReservation) throws PlanningException, ContractValidationException {
    // Adjust the ResourceDefinition to account for system "imperfections"
    // (e.g., scheduling delays for large containers).
    ReservationDefinition adjustedContract = adjustContract(plan, contract);
    // Compute the job allocation
    RLESparseResourceAllocation allocation = computeJobAllocation(plan, reservationId, adjustedContract, user);
    // If no job allocation was found, fail
    if (allocation == null) {
        throw new PlanningException("The planning algorithm could not find a valid allocation" + " for your request");
    }
    // Translate the allocation to a map (with zero paddings)
    long step = plan.getStep();
    long jobArrival = stepRoundUp(adjustedContract.getArrival(), step);
    long jobDeadline = stepRoundUp(adjustedContract.getDeadline(), step);
    Map<ReservationInterval, Resource> mapAllocations = allocationsToPaddedMap(allocation, jobArrival, jobDeadline);
    // Create the reservation
    ReservationAllocation capReservation = new // ID
    InMemoryReservationAllocation(// ID
    reservationId, // Contract
    adjustedContract, // User name
    user, // Queue name
    plan.getQueueName(), // Earliest start time
    findEarliestTime(mapAllocations), // Latest end time
    findLatestTime(mapAllocations), // Allocations
    mapAllocations, // Resource calculator
    plan.getResourceCalculator(), // Minimum allocation
    plan.getMinimumAllocation());
    // Add (or update) the reservation allocation
    if (oldReservation != null) {
        return plan.updateReservation(capReservation);
    } else {
        return plan.addReservation(capReservation, false);
    }
}

Example 9 with ReservationInterval

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval in project hadoop by apache.

the class PlanningAlgorithm method allocationsToPaddedMap.

private Map<ReservationInterval, Resource> allocationsToPaddedMap(RLESparseResourceAllocation allocation, long jobArrival, long jobDeadline) {
    // Allocate
    Map<ReservationInterval, Resource> mapAllocations = allocation.toIntervalMap();
    // Zero allocation
    Resource zeroResource = Resource.newInstance(0, 0);
    // Pad at the beginning
    long earliestStart = findEarliestTime(mapAllocations);
    if (jobArrival < earliestStart) {
        mapAllocations.put(new ReservationInterval(jobArrival, earliestStart), zeroResource);
    }
    // Pad at the beginning
    long latestEnd = findLatestTime(mapAllocations);
    if (latestEnd < jobDeadline) {
        mapAllocations.put(new ReservationInterval(latestEnd, jobDeadline), zeroResource);
    }
    return mapAllocations;
}

Example 10 with ReservationInterval

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval in project hadoop by apache.

the class StageAllocatorLowCostAligned method computeStageAllocation.

// computeJobAllocation()
@Override
public Map<ReservationInterval, Resource> computeStageAllocation(Plan plan, Map<Long, Resource> planLoads, RLESparseResourceAllocation planModifications, ReservationRequest rr, long stageEarliestStart, long stageDeadline, String user, ReservationId oldId) {
    // Initialize
    ResourceCalculator resCalc = plan.getResourceCalculator();
    Resource capacity = plan.getTotalCapacity();
    long step = plan.getStep();
    // Create allocationRequestsearlies
    RLESparseResourceAllocation allocationRequests = new RLESparseResourceAllocation(plan.getResourceCalculator());
    // Initialize parameters
    long duration = stepRoundUp(rr.getDuration(), step);
    int windowSizeInDurations = (int) ((stageDeadline - stageEarliestStart) / duration);
    int totalGangs = rr.getNumContainers() / rr.getConcurrency();
    int numContainersPerGang = rr.getConcurrency();
    Resource gang = Resources.multiply(rr.getCapability(), numContainersPerGang);
    // Set maxGangsPerUnit
    int maxGangsPerUnit = (int) Math.max(Math.floor(((double) totalGangs) / windowSizeInDurations), 1);
    maxGangsPerUnit = Math.max(maxGangsPerUnit / smoothnessFactor, 1);
    // If window size is too small, return null
    if (windowSizeInDurations <= 0) {
        return null;
    }
    // Initialize tree sorted by costs
    TreeSet<DurationInterval> durationIntervalsSortedByCost = new TreeSet<DurationInterval>(new Comparator<DurationInterval>() {

        @Override
        public int compare(DurationInterval val1, DurationInterval val2) {
            int cmp = Double.compare(val1.getTotalCost(), val2.getTotalCost());
            if (cmp != 0) {
                return cmp;
            }
            return (-1) * Long.compare(val1.getEndTime(), val2.getEndTime());
        }
    });
    // Add durationIntervals that end at (endTime - n*duration) for some n.
    for (long intervalEnd = stageDeadline; intervalEnd >= stageEarliestStart + duration; intervalEnd -= duration) {
        long intervalStart = intervalEnd - duration;
        // Get duration interval [intervalStart,intervalEnd)
        DurationInterval durationInterval = getDurationInterval(intervalStart, intervalEnd, planLoads, planModifications, capacity, resCalc, step);
        // If the interval can fit a gang, add it to the tree
        if (durationInterval.canAllocate(gang, capacity, resCalc)) {
            durationIntervalsSortedByCost.add(durationInterval);
        }
    }
    // Allocate
    int remainingGangs = totalGangs;
    while (remainingGangs > 0) {
        // If no durationInterval can fit a gang, break and return null
        if (durationIntervalsSortedByCost.isEmpty()) {
            break;
        }
        // Get best duration interval
        DurationInterval bestDurationInterval = durationIntervalsSortedByCost.first();
        int numGangsToAllocate = Math.min(maxGangsPerUnit, remainingGangs);
        numGangsToAllocate = Math.min(numGangsToAllocate, bestDurationInterval.numCanFit(gang, capacity, resCalc));
        // Add it
        remainingGangs -= numGangsToAllocate;
        ReservationInterval reservationInt = new ReservationInterval(bestDurationInterval.getStartTime(), bestDurationInterval.getEndTime());
        Resource reservationRes = Resources.multiply(rr.getCapability(), rr.getConcurrency() * numGangsToAllocate);
        planModifications.addInterval(reservationInt, reservationRes);
        allocationRequests.addInterval(reservationInt, reservationRes);
        // Remove from tree
        durationIntervalsSortedByCost.remove(bestDurationInterval);
        // Get updated interval
        DurationInterval updatedDurationInterval = getDurationInterval(bestDurationInterval.getStartTime(), bestDurationInterval.getStartTime() + duration, planLoads, planModifications, capacity, resCalc, step);
        // Add to tree, if possible
        if (updatedDurationInterval.canAllocate(gang, capacity, resCalc)) {
            durationIntervalsSortedByCost.add(updatedDurationInterval);
        }
    }
    // Get the final allocation
    Map<ReservationInterval, Resource> allocations = allocationRequests.toIntervalMap();
    // If no gangs are left to place we succeed and return the allocation
    if (remainingGangs <= 0) {
        return allocations;
    } else {
        // We remove unwanted side-effect from planModifications (needed for ANY).
        for (Map.Entry<ReservationInterval, Resource> tempAllocation : allocations.entrySet()) {
            planModifications.removeInterval(tempAllocation.getKey(), tempAllocation.getValue());
        }
        // Return null to signal failure in this allocation
        return null;
    }
}