Examples with ReservationDefinition org.apache.hadoop.yarn.api.records.ReservationDefinition used on opensource projects

Example 66 with ReservationDefinition

use of org.apache.hadoop.yarn.api.records.ReservationDefinition in project hadoop by apache.

the class TestReservationSystemWithRMHA method testUpdateReservationAndCheckAfterFailover.

@Test
public void testUpdateReservationAndCheckAfterFailover() throws Exception {
    startRMs();
    addNodeCapacityToPlan(rm1, 102400, 100);
    ClientRMService clientService = rm1.getClientRMService();
    ReservationId reservationID = getNewReservation(clientService).getReservationId();
    // create a reservation
    ReservationSubmissionRequest request = createReservationSubmissionRequest(reservationID);
    ReservationSubmissionResponse response = null;
    try {
        response = clientService.submitReservation(request);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(response);
    Assert.assertNotNull(reservationID);
    LOG.info("Submit reservation response: " + reservationID);
    ReservationDefinition reservationDefinition = request.getReservationDefinition();
    // Change any field
    long newDeadline = reservationDefinition.getDeadline() + 100;
    reservationDefinition.setDeadline(newDeadline);
    ReservationUpdateRequest updateRequest = ReservationUpdateRequest.newInstance(reservationDefinition, reservationID);
    rm1.updateReservationState(updateRequest);
    // Do the failover
    explicitFailover();
    rm2.registerNode("127.0.0.1:1", 102400, 100);
    RMState state = rm2.getRMContext().getStateStore().loadState();
    Map<ReservationId, ReservationAllocationStateProto> reservationStateMap = state.getReservationState().get(ReservationSystemTestUtil.reservationQ);
    Assert.assertNotNull(reservationStateMap);
    ReservationAllocationStateProto reservationState = reservationStateMap.get(reservationID);
    Assert.assertEquals(newDeadline, reservationState.getReservationDefinition().getDeadline());
}

Example 67 with ReservationDefinition

use of org.apache.hadoop.yarn.api.records.ReservationDefinition in project hadoop by apache.

the class TestReservationSystemWithRMHA method testSubmitReservationFailoverAndUpdate.

@Test
public void testSubmitReservationFailoverAndUpdate() throws Exception {
    startRMs();
    addNodeCapacityToPlan(rm1, 102400, 100);
    ClientRMService clientService = rm1.getClientRMService();
    ReservationId reservationID = getNewReservation(clientService).getReservationId();
    // create a reservation
    ReservationSubmissionRequest request = createReservationSubmissionRequest(reservationID);
    ReservationSubmissionResponse response = null;
    try {
        response = clientService.submitReservation(request);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(response);
    Assert.assertNotNull(reservationID);
    LOG.info("Submit reservation response: " + reservationID);
    ReservationDefinition reservationDefinition = request.getReservationDefinition();
    // Do the failover
    explicitFailover();
    addNodeCapacityToPlan(rm2, 102400, 100);
    // check if reservation exists after failover
    Plan plan = rm2.getRMContext().getReservationSystem().getPlan(ReservationSystemTestUtil.reservationQ);
    validateReservation(plan, reservationID, reservationDefinition);
    // update the reservation
    long newDeadline = reservationDefinition.getDeadline() + 100;
    reservationDefinition.setDeadline(newDeadline);
    ReservationUpdateRequest updateRequest = ReservationUpdateRequest.newInstance(reservationDefinition, reservationID);
    ReservationUpdateResponse updateResponse = null;
    clientService = rm2.getClientRMService();
    try {
        updateResponse = clientService.updateReservation(updateRequest);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(updateResponse);
    validateReservation(plan, reservationID, reservationDefinition);
}

Example 68 with ReservationDefinition

use of org.apache.hadoop.yarn.api.records.ReservationDefinition in project hadoop by apache.

the class RMStateStoreTestBase method testReservationStateStore.

public void testReservationStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int start = 1;
    int[] alloc = { 10, 10, 10, 10, 10 };
    ResourceCalculator res = new DefaultResourceCalculator();
    Resource minAlloc = Resource.newInstance(1024, 1);
    boolean hasGang = true;
    String planName = "dedicated";
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    ReservationAllocation allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    assertAllocationStateEqual(allocation, allocationStateProto);
    // 1. Load empty store and verify no errors
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<String, Map<ReservationId, ReservationAllocationStateProto>> reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    // 2. Store single reservation and verify
    String reservationIdName = r1.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify new state
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 3. update state test
    alloc = new int[] { 6, 6, 6 };
    hasGang = false;
    allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 2, 2 + alloc.length, ReservationSystemTestUtil.generateAllocation(1L, 2L, alloc), res, minAlloc, hasGang);
    allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify updated reservation
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 4. add a second one and remove the first one
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    ReservationAllocation allocation2 = new InMemoryReservationAllocation(r2, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto2 = ReservationSystemUtil.buildStateProto(allocation2);
    String reservationIdName2 = r2.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto2, planName, reservationIdName2);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    // load state and verify r1 is removed and r2 is still there
    Map<ReservationId, ReservationAllocationStateProto> reservations;
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNotNull(reservations);
    ReservationAllocationStateProto storedReservationAllocation = reservations.get(r1);
    Assert.assertNull("Removed reservation should not be available in store", storedReservationAllocation);
    storedReservationAllocation = reservations.get(r2);
    assertAllocationStateEqual(allocationStateProto2, storedReservationAllocation);
    assertAllocationStateEqual(allocation2, storedReservationAllocation);
    // 5. remove last reservation removes the plan state
    rmContext.getStateStore().removeReservation(planName, reservationIdName2);
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNull(reservations);
}

Example 69 with ReservationDefinition

use of org.apache.hadoop.yarn.api.records.ReservationDefinition in project hadoop by apache.

the class TestYarnClient method testUpdateReservation.

@Test
public void testUpdateReservation() throws Exception {
    MiniYARNCluster cluster = setupMiniYARNCluster();
    YarnClient client = setupYarnClient(cluster);
    try {
        Clock clock = new UTCClock();
        long arrival = clock.getTime();
        long duration = 60000;
        long deadline = (long) (arrival + 1.05 * duration);
        ReservationSubmissionRequest sRequest = submitReservationTestHelper(client, arrival, deadline, duration);
        ReservationDefinition rDef = sRequest.getReservationDefinition();
        ReservationRequest rr = rDef.getReservationRequests().getReservationResources().get(0);
        ReservationId reservationID = sRequest.getReservationId();
        rr.setNumContainers(5);
        arrival = clock.getTime();
        duration = 30000;
        deadline = (long) (arrival + 1.05 * duration);
        rr.setDuration(duration);
        rDef.setArrival(arrival);
        rDef.setDeadline(deadline);
        ReservationUpdateRequest uRequest = ReservationUpdateRequest.newInstance(rDef, reservationID);
        ReservationUpdateResponse uResponse = client.updateReservation(uRequest);
        Assert.assertNotNull(uResponse);
        System.out.println("Update reservation response: " + uResponse);
    } finally {
        // clean-up
        if (client != null) {
            client.stop();
        }
        cluster.stop();
    }
}

Example 70 with ReservationDefinition

use of org.apache.hadoop.yarn.api.records.ReservationDefinition in project hadoop by apache.

the class TestYarnClient method createSimpleReservationRequest.

private ReservationSubmissionRequest createSimpleReservationRequest(ReservationId reservationId, int numContainers, long arrival, long deadline, long duration) {
    // create a request with a single atomic ask
    ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration);
    ReservationRequests reqs = ReservationRequests.newInstance(Collections.singletonList(r), ReservationRequestInterpreter.R_ALL);
    ReservationDefinition rDef = ReservationDefinition.newInstance(arrival, deadline, reqs, "testYarnClient#reservation");
    ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, ReservationSystemTestUtil.reservationQ, reservationId);
    return request;
}