Examples with ReservationRequest org.apache.hadoop.yarn.api.records.ReservationRequest used on opensource projects

Example 16 with ReservationRequest

use of org.apache.hadoop.yarn.api.records.ReservationRequest in project hadoop by apache.

the class StageEarliestStartByDemand method setEarliestStartTime.

@Override
public long setEarliestStartTime(Plan plan, ReservationDefinition reservation, int index, ReservationRequest current, long stageDeadline) {
    step = plan.getStep();
    // If this is the first stage, don't bother with the computation.
    if (index < 1) {
        return reservation.getArrival();
    }
    // Get iterator
    ListIterator<ReservationRequest> li = reservation.getReservationRequests().getReservationResources().listIterator(index);
    ReservationRequest rr;
    // Calculate the total weight & total duration
    double totalWeight = calcWeight(current);
    long totalDuration = getRoundedDuration(current, plan);
    while (li.hasPrevious()) {
        rr = li.previous();
        totalWeight += calcWeight(rr);
        totalDuration += getRoundedDuration(rr, plan);
    }
    // Compute the weight of the current stage as compared to remaining ones
    double ratio = calcWeight(current) / totalWeight;
    // Estimate an early start time, such that:
    // 1. Every stage is guaranteed to receive at least its duration
    // 2. The remainder of the window is divided between stages
    // proportionally to its workload (total memory consumption)
    long window = stageDeadline - reservation.getArrival();
    long windowRemainder = window - totalDuration;
    long earlyStart = (long) (stageDeadline - getRoundedDuration(current, plan) - (windowRemainder * ratio));
    // Realign if necessary (since we did some arithmetic)
    earlyStart = stepRoundUp(earlyStart, step);
    // Return
    return earlyStart;
}

Example 17 with ReservationRequest

use of org.apache.hadoop.yarn.api.records.ReservationRequest in project hadoop by apache.

the class IterativePlanner method computeJobAllocation.

@Override
public RLESparseResourceAllocation computeJobAllocation(Plan plan, ReservationId reservationId, ReservationDefinition reservation, String user) throws PlanningException {
    // Initialize
    initialize(plan, reservationId, reservation);
    // Create the allocations data structure
    RLESparseResourceAllocation allocations = new RLESparseResourceAllocation(plan.getResourceCalculator());
    StageProvider stageProvider = new StageProvider(allocateLeft, reservation);
    // Current stage
    ReservationRequest currentReservationStage;
    // Stage deadlines
    long stageDeadline = stepRoundDown(reservation.getDeadline(), step);
    long successorStartingTime = -1;
    long predecessorEndTime = stepRoundDown(reservation.getArrival(), step);
    long stageArrivalTime = -1;
    // Iterate the stages in reverse order
    while (stageProvider.hasNext()) {
        // Get current stage
        currentReservationStage = stageProvider.next();
        // Validate that the ReservationRequest respects basic constraints
        validateInputStage(plan, currentReservationStage);
        if (allocateLeft) {
            stageArrivalTime = predecessorEndTime;
        } else {
            stageArrivalTime = reservation.getArrival();
            if (jobType == ReservationRequestInterpreter.R_ORDER || jobType == ReservationRequestInterpreter.R_ORDER_NO_GAP) {
                stageArrivalTime = computeEarliestStartingTime(plan, reservation, stageProvider.getCurrentIndex(), currentReservationStage, stageDeadline);
            }
            stageArrivalTime = stepRoundUp(stageArrivalTime, step);
            stageArrivalTime = Math.max(stageArrivalTime, reservation.getArrival());
        }
        // Compute the allocation of a single stage
        Map<ReservationInterval, Resource> curAlloc = computeStageAllocation(plan, currentReservationStage, stageArrivalTime, stageDeadline, user, reservationId);
        // (unless it's an ANY job, then we simply continue).
        if (curAlloc == null) {
            // If it's an ANY job, we can move to the next possible request
            if (jobType == ReservationRequestInterpreter.R_ANY) {
                continue;
            }
            // Otherwise, the job cannot be allocated
            throw new PlanningException("The request cannot be satisfied");
        }
        // Get the start & end time of the current allocation
        Long stageStartTime = findEarliestTime(curAlloc);
        Long stageEndTime = findLatestTime(curAlloc);
        // If we did find an allocation for the stage, add it
        for (Entry<ReservationInterval, Resource> entry : curAlloc.entrySet()) {
            allocations.addInterval(entry.getKey(), entry.getValue());
        }
        // If this is an ANY clause, we have finished
        if (jobType == ReservationRequestInterpreter.R_ANY) {
            break;
        }
        // If ORDER job, set the stageDeadline of the next stage to be processed
        if (jobType == ReservationRequestInterpreter.R_ORDER || jobType == ReservationRequestInterpreter.R_ORDER_NO_GAP) {
            // note that the test is different left-to-right and right-to-left
            if (jobType == ReservationRequestInterpreter.R_ORDER_NO_GAP && successorStartingTime != -1 && ((allocateLeft && predecessorEndTime < stageStartTime) || (!allocateLeft && (stageEndTime < successorStartingTime))) || (!isNonPreemptiveAllocation(curAlloc))) {
                throw new PlanningException("The allocation found does not respect ORDER_NO_GAP");
            }
            if (allocateLeft) {
                // Store the stageStartTime and set the new stageDeadline
                predecessorEndTime = stageEndTime;
            } else {
                // Store the stageStartTime and set the new stageDeadline
                successorStartingTime = stageStartTime;
                stageDeadline = stageStartTime;
            }
        }
    }
    // If the allocation is empty, return an error
    if (allocations.isEmpty()) {
        throw new PlanningException("The request cannot be satisfied");
    }
    return allocations;
}

Example 18 with ReservationRequest

use of org.apache.hadoop.yarn.api.records.ReservationRequest in project hadoop by apache.

the class TestGreedyReservationAgent method testAllImpossible.

@Test
public void testAllImpossible() throws PlanningException {
    prepareBasicPlan();
    // create an ALL request, with an impossible combination, it should be
    // rejected, and allocation remain unchanged
    ReservationDefinition rr = new ReservationDefinitionPBImpl();
    rr.setArrival(100L);
    rr.setDeadline(120L);
    ReservationRequests reqs = new ReservationRequestsPBImpl();
    reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
    ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), 55, 5, 10);
    ReservationRequest r2 = ReservationRequest.newInstance(Resource.newInstance(2048, 2), 55, 5, 20);
    List<ReservationRequest> list = new ArrayList<ReservationRequest>();
    list.add(r);
    list.add(r2);
    reqs.setReservationResources(list);
    rr.setReservationRequests(reqs);
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    boolean result = false;
    try {
        // submit to agent
        result = agent.createReservation(reservationID, "u1", plan, rr);
        fail();
    } catch (PlanningException p) {
    // expected
    }
    // validate results, we expect the second one to be accepted
    assertFalse("Agent-based allocation failed", result);
    assertTrue("Agent-based allocation failed", plan.getAllReservations().size() == 2);
    System.out.println("--------AFTER ALL IMPOSSIBLE ALLOCATION (queue: " + reservationID + ")----------");
    System.out.println(plan.toString());
    System.out.println(plan.toCumulativeString());
}

Example 19 with ReservationRequest

use of org.apache.hadoop.yarn.api.records.ReservationRequest in project hadoop by apache.

the class TestGreedyReservationAgent method testOrderNoGap.

@Test
public void testOrderNoGap() throws PlanningException {
    prepareBasicPlan();
    // create a chain of 4 RR, mixing gang and non-gang
    ReservationDefinition rr = new ReservationDefinitionPBImpl();
    rr.setArrival(0 * step);
    rr.setDeadline(60 * step);
    ReservationRequests reqs = new ReservationRequestsPBImpl();
    reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER_NO_GAP);
    ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(2048, 2), 10, 1, 10 * step);
    ReservationRequest r2 = ReservationRequest.newInstance(Resource.newInstance(1024, 1), 10, 10, 20 * step);
    List<ReservationRequest> list = new ArrayList<ReservationRequest>();
    list.add(r);
    list.add(r2);
    list.add(r);
    list.add(r2);
    reqs.setReservationResources(list);
    rr.setReservationRequests(reqs);
    rr.setReservationRequests(reqs);
    // submit to agent
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    agent.createReservation(reservationID, "u1", plan, rr);
    System.out.println("--------AFTER ORDER ALLOCATION (queue: " + reservationID + ")----------");
    System.out.println(plan.toString());
    System.out.println(plan.toCumulativeString());
    // validate
    assertTrue("Agent-based allocation failed", reservationID != null);
    assertTrue("Agent-based allocation failed", plan.getAllReservations().size() == 3);
    ReservationAllocation cs = plan.getReservationById(reservationID);
    assertTrue(cs.toString(), check(cs, 0 * step, 10 * step, 20, 1024, 1));
    assertTrue(cs.toString(), check(cs, 10 * step, 30 * step, 10, 1024, 1));
    assertTrue(cs.toString(), check(cs, 30 * step, 40 * step, 20, 1024, 1));
    assertTrue(cs.toString(), check(cs, 40 * step, 60 * step, 10, 1024, 1));
}

Example 20 with ReservationRequest

use of org.apache.hadoop.yarn.api.records.ReservationRequest in project hadoop by apache.

the class TestGreedyReservationAgent method testAnyImpossible.

@Test
public void testAnyImpossible() throws PlanningException {
    prepareBasicPlan();
    // create an ANY request, with all impossible alternatives
    ReservationDefinition rr = new ReservationDefinitionPBImpl();
    rr.setArrival(100L);
    rr.setDeadline(120L);
    ReservationRequests reqs = new ReservationRequestsPBImpl();
    reqs.setInterpreter(ReservationRequestInterpreter.R_ANY);
    // longer than arrival-deadline
    ReservationRequest r1 = ReservationRequest.newInstance(Resource.newInstance(1024, 1), 35, 5, 30);
    // above max cluster size
    ReservationRequest r2 = ReservationRequest.newInstance(Resource.newInstance(1024, 1), 110, 110, 10);
    List<ReservationRequest> list = new ArrayList<ReservationRequest>();
    list.add(r1);
    list.add(r2);
    reqs.setReservationResources(list);
    rr.setReservationRequests(reqs);
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    boolean result = false;
    try {
        // submit to agent
        result = agent.createReservation(reservationID, "u1", plan, rr);
        fail();
    } catch (PlanningException p) {
    // expected
    }
    // validate results, we expect the second one to be accepted
    assertFalse("Agent-based allocation should have failed", result);
    assertTrue("Agent-based allocation should have failed", plan.getAllReservations().size() == 2);
    System.out.println("--------AFTER ANY IMPOSSIBLE ALLOCATION (queue: " + reservationID + ")----------");
    System.out.println(plan.toString());
    System.out.println(plan.toCumulativeString());
}