Examples with ReservationAllocationStateProto org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto used on opensource projects

Example 6 with ReservationAllocationStateProto

use of org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto in project hadoop by apache.

the class ZKRMStateStore method loadReservationSystemState.

private void loadReservationSystemState(RMState rmState) throws Exception {
    List<String> planNodes = getChildren(reservationRoot);
    for (String planName : planNodes) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Loading plan from znode: " + planName);
        }
        String planNodePath = getNodePath(reservationRoot, planName);
        List<String> reservationNodes = getChildren(planNodePath);
        for (String reservationNodeName : reservationNodes) {
            String reservationNodePath = getNodePath(planNodePath, reservationNodeName);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Loading reservation from znode: " + reservationNodePath);
            }
            byte[] reservationData = getData(reservationNodePath);
            ReservationAllocationStateProto allocationState = ReservationAllocationStateProto.parseFrom(reservationData);
            if (!rmState.getReservationState().containsKey(planName)) {
                rmState.getReservationState().put(planName, new HashMap<>());
            }
            ReservationId reservationId = ReservationId.parseReservationId(reservationNodeName);
            rmState.getReservationState().get(planName).put(reservationId, allocationState);
        }
    }
}

Example 7 with ReservationAllocationStateProto

use of org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto in project hadoop by apache.

the class RMStateStoreTestBase method validateStoredReservation.

private void validateStoredReservation(RMStateStoreHelper stateStoreHelper, TestDispatcher dispatcher, RMContext rmContext, ReservationId r1, String planName, ReservationAllocation allocation, ReservationAllocationStateProto allocationStateProto) throws Exception {
    RMStateStore store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<String, Map<ReservationId, ReservationAllocationStateProto>> reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    Map<ReservationId, ReservationAllocationStateProto> reservations = reservationState.get(planName);
    Assert.assertNotNull(reservations);
    ReservationAllocationStateProto storedReservationAllocation = reservations.get(r1);
    Assert.assertNotNull(storedReservationAllocation);
    assertAllocationStateEqual(allocationStateProto, storedReservationAllocation);
    assertAllocationStateEqual(allocation, storedReservationAllocation);
}

Example 8 with ReservationAllocationStateProto

use of org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto in project hadoop by apache.

the class RMStateStoreTestBase method testReservationStateStore.

public void testReservationStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int start = 1;
    int[] alloc = { 10, 10, 10, 10, 10 };
    ResourceCalculator res = new DefaultResourceCalculator();
    Resource minAlloc = Resource.newInstance(1024, 1);
    boolean hasGang = true;
    String planName = "dedicated";
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    ReservationAllocation allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    assertAllocationStateEqual(allocation, allocationStateProto);
    // 1. Load empty store and verify no errors
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<String, Map<ReservationId, ReservationAllocationStateProto>> reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    // 2. Store single reservation and verify
    String reservationIdName = r1.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify new state
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 3. update state test
    alloc = new int[] { 6, 6, 6 };
    hasGang = false;
    allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 2, 2 + alloc.length, ReservationSystemTestUtil.generateAllocation(1L, 2L, alloc), res, minAlloc, hasGang);
    allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify updated reservation
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 4. add a second one and remove the first one
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    ReservationAllocation allocation2 = new InMemoryReservationAllocation(r2, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto2 = ReservationSystemUtil.buildStateProto(allocation2);
    String reservationIdName2 = r2.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto2, planName, reservationIdName2);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    // load state and verify r1 is removed and r2 is still there
    Map<ReservationId, ReservationAllocationStateProto> reservations;
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNotNull(reservations);
    ReservationAllocationStateProto storedReservationAllocation = reservations.get(r1);
    Assert.assertNull("Removed reservation should not be available in store", storedReservationAllocation);
    storedReservationAllocation = reservations.get(r2);
    assertAllocationStateEqual(allocationStateProto2, storedReservationAllocation);
    assertAllocationStateEqual(allocation2, storedReservationAllocation);
    // 5. remove last reservation removes the plan state
    rmContext.getStateStore().removeReservation(planName, reservationIdName2);
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNull(reservations);
}

Example 9 with ReservationAllocationStateProto

use of org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto in project hadoop by apache.

the class TestReservationSystemWithRMHA method testSubmitReservationAndCheckAfterFailover.

@Test
public void testSubmitReservationAndCheckAfterFailover() throws Exception {
    startRMs();
    addNodeCapacityToPlan(rm1, 102400, 100);
    ClientRMService clientService = rm1.getClientRMService();
    ReservationId reservationID = getNewReservation(clientService).getReservationId();
    // create a reservation
    ReservationSubmissionRequest request = createReservationSubmissionRequest(reservationID);
    ReservationSubmissionResponse response = null;
    try {
        response = clientService.submitReservation(request);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(response);
    Assert.assertNotNull(reservationID);
    LOG.info("Submit reservation response: " + reservationID);
    // Do the failover
    explicitFailover();
    rm2.registerNode("127.0.0.1:1", 102400, 100);
    RMState state = rm2.getRMContext().getStateStore().loadState();
    Map<ReservationId, ReservationAllocationStateProto> reservationStateMap = state.getReservationState().get(ReservationSystemTestUtil.reservationQ);
    Assert.assertNotNull(reservationStateMap);
    Assert.assertNotNull(reservationStateMap.get(reservationID));
}

Example 10 with ReservationAllocationStateProto

use of org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto in project hadoop by apache.

the class TestReservationSystemWithRMHA method testUpdateReservationAndCheckAfterFailover.

@Test
public void testUpdateReservationAndCheckAfterFailover() throws Exception {
    startRMs();
    addNodeCapacityToPlan(rm1, 102400, 100);
    ClientRMService clientService = rm1.getClientRMService();
    ReservationId reservationID = getNewReservation(clientService).getReservationId();
    // create a reservation
    ReservationSubmissionRequest request = createReservationSubmissionRequest(reservationID);
    ReservationSubmissionResponse response = null;
    try {
        response = clientService.submitReservation(request);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(response);
    Assert.assertNotNull(reservationID);
    LOG.info("Submit reservation response: " + reservationID);
    ReservationDefinition reservationDefinition = request.getReservationDefinition();
    // Change any field
    long newDeadline = reservationDefinition.getDeadline() + 100;
    reservationDefinition.setDeadline(newDeadline);
    ReservationUpdateRequest updateRequest = ReservationUpdateRequest.newInstance(reservationDefinition, reservationID);
    rm1.updateReservationState(updateRequest);
    // Do the failover
    explicitFailover();
    rm2.registerNode("127.0.0.1:1", 102400, 100);
    RMState state = rm2.getRMContext().getStateStore().loadState();
    Map<ReservationId, ReservationAllocationStateProto> reservationStateMap = state.getReservationState().get(ReservationSystemTestUtil.reservationQ);
    Assert.assertNotNull(reservationStateMap);
    ReservationAllocationStateProto reservationState = reservationStateMap.get(reservationID);
    Assert.assertEquals(newDeadline, reservationState.getReservationDefinition().getDeadline());
}