Examples with ReservationSchedulerConfiguration org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration used on opensource projects

Example 1 with ReservationSchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration in project hadoop by apache.

the class TestGreedyReservationAgent method setup.

@Before
public void setup() throws Exception {
    long seed = rand.nextLong();
    rand.setSeed(seed);
    Log.getLog().info("Running with seed: " + seed);
    // setting completely loose quotas
    long timeWindow = 1000000L;
    Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
    step = 1000L;
    String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName();
    float instConstraint = 100;
    float avgConstraint = 100;
    ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint);
    CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
    policy.init(reservationQ, conf);
    // setting conf to
    conf.setBoolean(GreedyReservationAgent.GREEDY_FAVOR_EARLY_ALLOCATION, allocateLeft);
    agent = new GreedyReservationAgent();
    agent.init(conf);
    QueueMetrics queueMetrics = mock(QueueMetrics.class);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true, context);
}

Example 2 with ReservationSchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration in project hadoop by apache.

the class TestGreedyReservationAgent method testStress.

public void testStress(int numJobs) throws PlanningException, IOException {
    long timeWindow = 1000000L;
    Resource clusterCapacity = Resource.newInstance(500 * 100 * 1024, 500 * 32);
    step = 1000L;
    ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
    CapacityScheduler scheduler = testUtil.mockCapacityScheduler(500 * 100);
    String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName();
    float instConstraint = 100;
    float avgConstraint = 100;
    ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint);
    CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
    policy.init(reservationQ, conf);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true, context);
    int acc = 0;
    List<ReservationDefinition> list = new ArrayList<ReservationDefinition>();
    for (long i = 0; i < numJobs; i++) {
        list.add(ReservationSystemTestUtil.generateRandomRR(rand, i));
    }
    long start = System.currentTimeMillis();
    for (int i = 0; i < numJobs; i++) {
        try {
            if (agent.createReservation(ReservationSystemTestUtil.getNewReservationId(), "u" + i % 100, plan, list.get(i))) {
                acc++;
            }
        } catch (PlanningException p) {
        // ignore exceptions
        }
    }
    long end = System.currentTimeMillis();
    System.out.println("Submitted " + numJobs + " jobs " + " accepted " + acc + " in " + (end - start) + "ms");
}

Example 3 with ReservationSchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration in project hadoop by apache.

the class TestGreedyReservationAgent method testSharingPolicyFeedback.

@SuppressWarnings("javadoc")
@Test
public void testSharingPolicyFeedback() throws PlanningException {
    prepareBasicPlan();
    // let's constraint the instantaneous allocation and see the
    // policy kicking in during planning
    float instConstraint = 40;
    float avgConstraint = 40;
    ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(plan.getQueueName(), 100000, instConstraint, avgConstraint);
    plan.getSharingPolicy().init(plan.getQueueName(), conf);
    // create a request with a single atomic ask
    ReservationDefinition rr = new ReservationDefinitionPBImpl();
    rr.setArrival(5 * step);
    rr.setDeadline(100 * step);
    ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(2048, 2), 20, 20, 10 * step);
    ReservationRequests reqs = new ReservationRequestsPBImpl();
    reqs.setReservationResources(Collections.singletonList(r));
    rr.setReservationRequests(reqs);
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    agent.createReservation(reservationID, "u3", plan, rr);
    ReservationId reservationID2 = ReservationSystemTestUtil.getNewReservationId();
    agent.createReservation(reservationID2, "u3", plan, rr);
    ReservationDefinition rr3 = new ReservationDefinitionPBImpl();
    rr3.setArrival(5 * step);
    rr3.setDeadline(100 * step);
    ReservationRequest r3 = ReservationRequest.newInstance(Resource.newInstance(2048, 2), 45, 45, 10 * step);
    ReservationRequests reqs3 = new ReservationRequestsPBImpl();
    reqs3.setReservationResources(Collections.singletonList(r3));
    rr3.setReservationRequests(reqs3);
    ReservationId reservationID3 = ReservationSystemTestUtil.getNewReservationId();
    try {
        // RR3 is simply too big to fit
        agent.createReservation(reservationID3, "u3", plan, rr3);
        fail();
    } catch (PlanningException pe) {
    // expected
    }
    assertTrue("Agent-based allocation failed", reservationID != null);
    assertTrue("Agent-based allocation failed", plan.getAllReservations().size() == 4);
    ReservationAllocation cs = plan.getReservationById(reservationID);
    ReservationAllocation cs2 = plan.getReservationById(reservationID2);
    ReservationAllocation cs3 = plan.getReservationById(reservationID3);
    assertNotNull(cs);
    assertNotNull(cs2);
    assertNull(cs3);
    System.out.println("--------AFTER SIMPLE ALLOCATION (queue: " + reservationID + ")----------");
    System.out.println(plan.toString());
    System.out.println(plan.toCumulativeString());
    if (allocateLeft) {
        for (long i = 5 * step; i < 15 * step; i++) {
            assertTrue("Agent-based allocation unexpected", Resources.equals(cs.getResourcesAtTime(i), Resource.newInstance(2048 * 20, 2 * 20)));
        }
        for (long i = 15 * step; i < 25 * step; i++) {
            // RR2 is pushed out by the presence of RR
            assertTrue("Agent-based allocation unexpected", Resources.equals(cs2.getResourcesAtTime(i), Resource.newInstance(2048 * 20, 2 * 20)));
        }
    } else {
        for (long i = 90 * step; i < 100 * step; i++) {
            assertTrue("Agent-based allocation unexpected", Resources.equals(cs.getResourcesAtTime(i), Resource.newInstance(2048 * 20, 2 * 20)));
        }
        for (long i = 80 * step; i < 90 * step; i++) {
            assertTrue("Agent-based allocation unexpected", Resources.equals(cs2.getResourcesAtTime(i), Resource.newInstance(2048 * 20, 2 * 20)));
        }
    }
}

Example 4 with ReservationSchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration in project hadoop by apache.

the class TestSimpleCapacityReplanner method testReplanningPlanCapacityLoss.

@Test
public void testReplanningPlanCapacityLoss() throws PlanningException {
    Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
    Resource minAlloc = Resource.newInstance(1024, 1);
    Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
    ResourceCalculator res = new DefaultResourceCalculator();
    long step = 1L;
    Clock clock = mock(Clock.class);
    ReservationAgent agent = mock(ReservationAgent.class);
    SharingPolicy policy = new NoOverCommitPolicy();
    policy.init("root.dedicated", null);
    QueueMetrics queueMetrics = mock(QueueMetrics.class);
    when(clock.getTime()).thenReturn(0L);
    SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    ReservationSchedulerConfiguration conf = mock(ReservationSchedulerConfiguration.class);
    when(conf.getEnforcementWindow(any(String.class))).thenReturn(6L);
    enf.init("blah", conf);
    // Initialize the plan with more resources
    InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true, context, clock);
    // add reservation filling the plan (separating them 1ms, so we are sure
    // s2 follows s1 on acceptance
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int[] f5 = { 20, 20, 20, 20, 20 };
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(0, 0 + f5.length, f5.length);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, rDef, "u3", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(1L);
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, rDef, "u4", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(2L);
    ReservationId r3 = ReservationId.newInstance(ts, 3);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, rDef, "u5", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(3L);
    ReservationId r4 = ReservationId.newInstance(ts, 4);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r4, rDef, "u6", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(4L);
    ReservationId r5 = ReservationId.newInstance(ts, 5);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r5, rDef, "u7", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    int[] f6 = { 50, 50, 50, 50, 50 };
    ReservationId r6 = ReservationId.newInstance(ts, 6);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r6, rDef, "u3", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc), false));
    when(clock.getTime()).thenReturn(6L);
    ReservationId r7 = ReservationId.newInstance(ts, 7);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r7, rDef, "u4", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc), false));
    // remove some of the resources (requires replanning)
    plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70));
    when(clock.getTime()).thenReturn(0L);
    // run the replanner
    enf.plan(plan, null);
    // check which reservation are still present
    assertNotNull(plan.getReservationById(r1));
    assertNotNull(plan.getReservationById(r2));
    assertNotNull(plan.getReservationById(r3));
    assertNotNull(plan.getReservationById(r6));
    assertNotNull(plan.getReservationById(r7));
    // and which ones are removed
    assertNull(plan.getReservationById(r4));
    assertNull(plan.getReservationById(r5));
    // check resources at each moment in time no more exceed capacity
    for (int i = 0; i < 20; i++) {
        long tot = 0;
        for (ReservationAllocation r : plan.getReservationsAtTime(i)) {
            tot = r.getResourcesAtTime(i).getMemorySize();
        }
        assertTrue(tot <= 70 * 1024);
    }
}

Example 5 with ReservationSchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration in project hadoop by apache.

the class TestAlignedPlanner method setup.

@Before
public void setup() throws Exception {
    // Initialize random seed
    long seed = rand.nextLong();
    rand.setSeed(seed);
    Log.getLog().info("Running with seed: " + seed);
    // Set cluster parameters
    long timeWindow = 1000000L;
    int capacityMem = 100 * 1024;
    int capacityCores = 100;
    step = 60000L;
    Resource clusterCapacity = Resource.newInstance(capacityMem, capacityCores);
    String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName();
    float instConstraint = 100;
    float avgConstraint = 100;
    ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint);
    CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
    policy.init(reservationQ, conf);
    QueueMetrics queueMetrics = mock(QueueMetrics.class);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    conf.setInt(AlignedPlannerWithGreedy.SMOOTHNESS_FACTOR, AlignedPlannerWithGreedy.DEFAULT_SMOOTHNESS_FACTOR);
    // Set planning agent
    agent = new AlignedPlannerWithGreedy();
    agent.init(conf);
    // Create Plan
    plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true, context);
}