Examples with ReservationAllocation org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation used on opensource projects

Example 16 with ReservationAllocation

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation in project hadoop by apache.

the class TestSimpleCapacityReplanner method testReplanningPlanCapacityLoss.

@Test
public void testReplanningPlanCapacityLoss() throws PlanningException {
    Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
    Resource minAlloc = Resource.newInstance(1024, 1);
    Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
    ResourceCalculator res = new DefaultResourceCalculator();
    long step = 1L;
    Clock clock = mock(Clock.class);
    ReservationAgent agent = mock(ReservationAgent.class);
    SharingPolicy policy = new NoOverCommitPolicy();
    policy.init("root.dedicated", null);
    QueueMetrics queueMetrics = mock(QueueMetrics.class);
    when(clock.getTime()).thenReturn(0L);
    SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    ReservationSchedulerConfiguration conf = mock(ReservationSchedulerConfiguration.class);
    when(conf.getEnforcementWindow(any(String.class))).thenReturn(6L);
    enf.init("blah", conf);
    // Initialize the plan with more resources
    InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true, context, clock);
    // add reservation filling the plan (separating them 1ms, so we are sure
    // s2 follows s1 on acceptance
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int[] f5 = { 20, 20, 20, 20, 20 };
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(0, 0 + f5.length, f5.length);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, rDef, "u3", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(1L);
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, rDef, "u4", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(2L);
    ReservationId r3 = ReservationId.newInstance(ts, 3);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, rDef, "u5", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(3L);
    ReservationId r4 = ReservationId.newInstance(ts, 4);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r4, rDef, "u6", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(4L);
    ReservationId r5 = ReservationId.newInstance(ts, 5);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r5, rDef, "u7", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    int[] f6 = { 50, 50, 50, 50, 50 };
    ReservationId r6 = ReservationId.newInstance(ts, 6);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r6, rDef, "u3", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc), false));
    when(clock.getTime()).thenReturn(6L);
    ReservationId r7 = ReservationId.newInstance(ts, 7);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r7, rDef, "u4", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc), false));
    // remove some of the resources (requires replanning)
    plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70));
    when(clock.getTime()).thenReturn(0L);
    // run the replanner
    enf.plan(plan, null);
    // check which reservation are still present
    assertNotNull(plan.getReservationById(r1));
    assertNotNull(plan.getReservationById(r2));
    assertNotNull(plan.getReservationById(r3));
    assertNotNull(plan.getReservationById(r6));
    assertNotNull(plan.getReservationById(r7));
    // and which ones are removed
    assertNull(plan.getReservationById(r4));
    assertNull(plan.getReservationById(r5));
    // check resources at each moment in time no more exceed capacity
    for (int i = 0; i < 20; i++) {
        long tot = 0;
        for (ReservationAllocation r : plan.getReservationsAtTime(i)) {
            tot = r.getResourcesAtTime(i).getMemorySize();
        }
        assertTrue(tot <= 70 * 1024);
    }
}

Example 17 with ReservationAllocation

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation in project hadoop by apache.

the class TestAlignedPlanner method testCostFunction.

@Test
public void testCostFunction() throws PlanningException {
    // Create large memory reservation
    ReservationDefinition rr7Mem1Core = createReservationDefinition(// Job arrival time
    10 * step, // Job deadline
    11 * step, new ReservationRequest[] { ReservationRequest.newInstance(// Capability
    Resource.newInstance(7 * 1024, 1), // Num containers
    1, // Concurrency
    1, // Duration
    step) }, ReservationRequestInterpreter.R_ALL, "u1");
    // Create reservation
    ReservationDefinition rr6Mem6Cores = createReservationDefinition(// Job arrival time
    10 * step, // Job deadline
    11 * step, new ReservationRequest[] { ReservationRequest.newInstance(// Capability
    Resource.newInstance(6 * 1024, 6), // Num containers
    1, // Concurrency
    1, // Duration
    step) }, ReservationRequestInterpreter.R_ALL, "u2");
    // Create reservation
    ReservationDefinition rr = createReservationDefinition(// Job arrival time
    10 * step, // Job deadline
    12 * step, new ReservationRequest[] { ReservationRequest.newInstance(// Capability
    Resource.newInstance(1024, 1), // Num containers
    1, // Concurrency
    1, // Duration
    step) }, ReservationRequestInterpreter.R_ALL, "u3");
    // Create reservation IDs
    ReservationId reservationID1 = ReservationSystemTestUtil.getNewReservationId();
    ReservationId reservationID2 = ReservationSystemTestUtil.getNewReservationId();
    ReservationId reservationID3 = ReservationSystemTestUtil.getNewReservationId();
    // Add all
    agent.createReservation(reservationID1, "u1", plan, rr7Mem1Core);
    agent.createReservation(reservationID2, "u2", plan, rr6Mem6Cores);
    agent.createReservation(reservationID3, "u3", plan, rr);
    // Get reservation
    ReservationAllocation alloc3 = plan.getReservationById(reservationID3);
    assertTrue(alloc3.toString(), check(alloc3, 10 * step, 11 * step, 0, 1024, 1));
    assertTrue(alloc3.toString(), check(alloc3, 11 * step, 12 * step, 1, 1024, 1));
}

Example 18 with ReservationAllocation

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation in project hadoop by apache.

the class TestAlignedPlanner method testUpdate.

@Test
public void testUpdate() throws PlanningException {
    // Create flexible reservation
    ReservationDefinition rrFlex = createReservationDefinition(// Job arrival time
    10 * step, // Job deadline
    14 * step, new ReservationRequest[] { ReservationRequest.newInstance(// Capability
    Resource.newInstance(1024, 1), // Num containers
    100, // Concurrency
    1, // Duration
    2 * step) }, ReservationRequestInterpreter.R_ALL, "u1");
    // Create blocking reservation
    ReservationDefinition rrBlock = createReservationDefinition(// Job arrival time
    10 * step, // Job deadline
    11 * step, new ReservationRequest[] { ReservationRequest.newInstance(// Capability
    Resource.newInstance(1024, 1), // Num containers
    100, // Concurrency
    100, // Duration
    step) }, ReservationRequestInterpreter.R_ALL, "u1");
    // Create reservation IDs
    ReservationId flexReservationID = ReservationSystemTestUtil.getNewReservationId();
    ReservationId blockReservationID = ReservationSystemTestUtil.getNewReservationId();
    // Add block, add flex, remove block, update flex
    agent.createReservation(blockReservationID, "uBlock", plan, rrBlock);
    agent.createReservation(flexReservationID, "uFlex", plan, rrFlex);
    agent.deleteReservation(blockReservationID, "uBlock", plan);
    agent.updateReservation(flexReservationID, "uFlex", plan, rrFlex);
    // CHECK: allocation was accepted
    assertTrue("Agent-based allocation failed", flexReservationID != null);
    assertTrue("Agent-based allocation failed", plan.getAllReservations().size() == 1);
    // Get reservation
    ReservationAllocation alloc1 = plan.getReservationById(flexReservationID);
    // Verify allocation
    assertTrue(alloc1.toString(), check(alloc1, 10 * step, 14 * step, 50, 1024, 1));
}

Example 19 with ReservationAllocation

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation in project hadoop by apache.

the class TestAlignedPlanner method testAnyAccept.

@Test
public void testAnyAccept() throws PlanningException {
    // Prepare basic plan
    int numJobsInScenario = initializeScenario2();
    // Create reservation
    ReservationDefinition rr1 = createReservationDefinition(// Job arrival time
    10 * step, // Job deadline
    15 * step, new ReservationRequest[] { ReservationRequest.newInstance(// Capability
    Resource.newInstance(1024, 1), // Num containers
    20, // Concurrency
    20, // Duration
    step), ReservationRequest.newInstance(// Capability
    Resource.newInstance(1024, 1), // Num containers
    20, // Concurrency
    20, // Duration
    2 * step) }, ReservationRequestInterpreter.R_ANY, "u1");
    // Add reservation
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    agent.createReservation(reservationID, "u1", plan, rr1);
    // CHECK: allocation was accepted
    assertTrue("Agent-based allocation failed", reservationID != null);
    assertTrue("Agent-based allocation failed", plan.getAllReservations().size() == numJobsInScenario + 1);
    // Get reservation
    ReservationAllocation alloc1 = plan.getReservationById(reservationID);
    // Verify allocation
    assertTrue(alloc1.toString(), check(alloc1, 14 * step, 15 * step, 20, 1024, 1));
}

Example 20 with ReservationAllocation

use of org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation in project hadoop by apache.

the class RMStateStoreTestBase method testReservationStateStore.

public void testReservationStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int start = 1;
    int[] alloc = { 10, 10, 10, 10, 10 };
    ResourceCalculator res = new DefaultResourceCalculator();
    Resource minAlloc = Resource.newInstance(1024, 1);
    boolean hasGang = true;
    String planName = "dedicated";
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    ReservationAllocation allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    assertAllocationStateEqual(allocation, allocationStateProto);
    // 1. Load empty store and verify no errors
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<String, Map<ReservationId, ReservationAllocationStateProto>> reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    // 2. Store single reservation and verify
    String reservationIdName = r1.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify new state
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 3. update state test
    alloc = new int[] { 6, 6, 6 };
    hasGang = false;
    allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 2, 2 + alloc.length, ReservationSystemTestUtil.generateAllocation(1L, 2L, alloc), res, minAlloc, hasGang);
    allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify updated reservation
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 4. add a second one and remove the first one
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    ReservationAllocation allocation2 = new InMemoryReservationAllocation(r2, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto2 = ReservationSystemUtil.buildStateProto(allocation2);
    String reservationIdName2 = r2.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto2, planName, reservationIdName2);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    // load state and verify r1 is removed and r2 is still there
    Map<ReservationId, ReservationAllocationStateProto> reservations;
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNotNull(reservations);
    ReservationAllocationStateProto storedReservationAllocation = reservations.get(r1);
    Assert.assertNull("Removed reservation should not be available in store", storedReservationAllocation);
    storedReservationAllocation = reservations.get(r2);
    assertAllocationStateEqual(allocationStateProto2, storedReservationAllocation);
    assertAllocationStateEqual(allocation2, storedReservationAllocation);
    // 5. remove last reservation removes the plan state
    rmContext.getStateStore().removeReservation(planName, reservationIdName2);
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNull(reservations);
}