Examples with RMContext org.apache.hadoop.yarn.server.resourcemanager.RMContext used on opensource projects

Example 51 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class RMStateStoreTestBase method testReservationStateStore.

public void testReservationStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int start = 1;
    int[] alloc = { 10, 10, 10, 10, 10 };
    ResourceCalculator res = new DefaultResourceCalculator();
    Resource minAlloc = Resource.newInstance(1024, 1);
    boolean hasGang = true;
    String planName = "dedicated";
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    ReservationAllocation allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    assertAllocationStateEqual(allocation, allocationStateProto);
    // 1. Load empty store and verify no errors
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<String, Map<ReservationId, ReservationAllocationStateProto>> reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    // 2. Store single reservation and verify
    String reservationIdName = r1.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify new state
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 3. update state test
    alloc = new int[] { 6, 6, 6 };
    hasGang = false;
    allocation = new InMemoryReservationAllocation(r1, rDef, "u3", planName, 2, 2 + alloc.length, ReservationSystemTestUtil.generateAllocation(1L, 2L, alloc), res, minAlloc, hasGang);
    allocationStateProto = ReservationSystemUtil.buildStateProto(allocation);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    rmContext.getStateStore().storeNewReservation(allocationStateProto, planName, reservationIdName);
    // load state and verify updated reservation
    validateStoredReservation(stateStoreHelper, dispatcher, rmContext, r1, planName, allocation, allocationStateProto);
    // 4. add a second one and remove the first one
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    ReservationAllocation allocation2 = new InMemoryReservationAllocation(r2, rDef, "u3", planName, 0, 0 + alloc.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, alloc), res, minAlloc, hasGang);
    ReservationAllocationStateProto allocationStateProto2 = ReservationSystemUtil.buildStateProto(allocation2);
    String reservationIdName2 = r2.toString();
    rmContext.getStateStore().storeNewReservation(allocationStateProto2, planName, reservationIdName2);
    rmContext.getStateStore().removeReservation(planName, reservationIdName);
    // load state and verify r1 is removed and r2 is still there
    Map<ReservationId, ReservationAllocationStateProto> reservations;
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNotNull(reservations);
    ReservationAllocationStateProto storedReservationAllocation = reservations.get(r1);
    Assert.assertNull("Removed reservation should not be available in store", storedReservationAllocation);
    storedReservationAllocation = reservations.get(r2);
    assertAllocationStateEqual(allocationStateProto2, storedReservationAllocation);
    assertAllocationStateEqual(allocation2, storedReservationAllocation);
    // 5. remove last reservation removes the plan state
    rmContext.getStateStore().removeReservation(planName, reservationIdName2);
    store = stateStoreHelper.getRMStateStore();
    when(rmContext.getStateStore()).thenReturn(store);
    store.setRMDispatcher(dispatcher);
    state = store.loadState();
    reservationState = state.getReservationState();
    Assert.assertNotNull(reservationState);
    reservations = reservationState.get(planName);
    Assert.assertNull(reservations);
}

Example 52 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class RMStateStoreTestBase method testRMAppStateStore.

void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, StoreStateVerifier verifier) throws Exception {
    long submitTime = System.currentTimeMillis();
    long startTime = System.currentTimeMillis() + 1234;
    Configuration conf = new YarnConfiguration();
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    AMRMTokenSecretManager appTokenMgr = spy(new AMRMTokenSecretManager(conf, rmContext));
    MasterKeyData masterKeyData = appTokenMgr.createNewMasterKey();
    when(appTokenMgr.getMasterKey()).thenReturn(masterKeyData);
    ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
    ApplicationAttemptId attemptId1 = ApplicationAttemptId.fromString("appattempt_1352994193343_0001_000001");
    ApplicationId appId1 = attemptId1.getApplicationId();
    storeApp(store, appId1, submitTime, startTime);
    verifier.afterStoreApp(store, appId1);
    // create application token and client token key for attempt1
    Token<AMRMTokenIdentifier> appAttemptToken1 = generateAMRMToken(attemptId1, appTokenMgr);
    SecretKey clientTokenKey1 = clientToAMTokenMgr.createMasterKey(attemptId1);
    ContainerId containerId1 = storeAttempt(store, attemptId1, "container_1352994193343_0001_01_000001", appAttemptToken1, clientTokenKey1, dispatcher).getMasterContainer().getId();
    String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
    ApplicationAttemptId attemptId2 = ApplicationAttemptId.fromString(appAttemptIdStr2);
    // create application token and client token key for attempt2
    Token<AMRMTokenIdentifier> appAttemptToken2 = generateAMRMToken(attemptId2, appTokenMgr);
    SecretKey clientTokenKey2 = clientToAMTokenMgr.createMasterKey(attemptId2);
    ContainerId containerId2 = storeAttempt(store, attemptId2, "container_1352994193343_0001_02_000001", appAttemptToken2, clientTokenKey2, dispatcher).getMasterContainer().getId();
    ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString("appattempt_1352994193343_0002_000001");
    ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
    storeApp(store, appIdRemoved, submitTime, startTime);
    storeAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null, null, dispatcher);
    verifier.afterStoreAppAttempt(store, attemptIdRemoved);
    RMApp mockRemovedApp = mock(RMApp.class);
    RMAppAttemptMetrics mockRmAppAttemptMetrics = mock(RMAppAttemptMetrics.class);
    HashMap<ApplicationAttemptId, RMAppAttempt> attempts = new HashMap<ApplicationAttemptId, RMAppAttempt>();
    ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
    context.setApplicationId(appIdRemoved);
    when(mockRemovedApp.getSubmitTime()).thenReturn(submitTime);
    when(mockRemovedApp.getApplicationSubmissionContext()).thenReturn(context);
    when(mockRemovedApp.getAppAttempts()).thenReturn(attempts);
    when(mockRemovedApp.getUser()).thenReturn("user1");
    RMAppAttempt mockRemovedAttempt = mock(RMAppAttempt.class);
    when(mockRemovedAttempt.getAppAttemptId()).thenReturn(attemptIdRemoved);
    when(mockRemovedAttempt.getRMAppAttemptMetrics()).thenReturn(mockRmAppAttemptMetrics);
    when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()).thenReturn(new AggregateAppResourceUsage(0, 0));
    attempts.put(attemptIdRemoved, mockRemovedAttempt);
    store.removeApplication(mockRemovedApp);
    // remove application directory recursively.
    storeApp(store, appIdRemoved, submitTime, startTime);
    storeAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null, null, dispatcher);
    store.removeApplication(mockRemovedApp);
    // let things settle down
    Thread.sleep(1000);
    store.close();
    // give tester a chance to modify app state in the store
    modifyAppState();
    // load state
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<ApplicationId, ApplicationStateData> rmAppState = state.getApplicationState();
    ApplicationStateData appState = rmAppState.get(appId1);
    // app is loaded
    assertNotNull(appState);
    // app is loaded correctly
    assertEquals(submitTime, appState.getSubmitTime());
    assertEquals(startTime, appState.getStartTime());
    // submission context is loaded correctly
    assertEquals(appId1, appState.getApplicationSubmissionContext().getApplicationId());
    ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
    // attempt1 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId1, attemptState.getAttemptId());
    assertEquals(-1000, attemptState.getAMContainerExitStatus());
    // attempt1 container is loaded correctly
    assertEquals(containerId1, attemptState.getMasterContainer().getId());
    // attempt1 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey1.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    assertEquals("context", appState.getCallerContext().getContext());
    attemptState = appState.getAttempt(attemptId2);
    // attempt2 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId2, attemptState.getAttemptId());
    // attempt2 container is loaded correctly
    assertEquals(containerId2, attemptState.getMasterContainer().getId());
    // attempt2 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey2.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    //******* update application/attempt state *******//
    ApplicationStateData appState2 = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), appState.getApplicationSubmissionContext(), RMAppState.FINISHED, "appDiagnostics", 1234, appState.getCallerContext());
    appState2.attempts.putAll(appState.attempts);
    store.updateApplicationState(appState2);
    ApplicationAttemptStateData oldAttemptState = attemptState;
    ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.newInstance(oldAttemptState.getAttemptId(), oldAttemptState.getMasterContainer(), oldAttemptState.getAppAttemptTokens(), oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 100, oldAttemptState.getFinishTime(), 0, 0, 0, 0);
    store.updateApplicationAttemptState(newAttemptState);
    // test updating the state of an app/attempt whose initial state was not
    // saved.
    ApplicationId dummyAppId = ApplicationId.newInstance(1234, 10);
    ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl();
    dummyContext.setApplicationId(dummyAppId);
    ApplicationStateData dummyApp = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), dummyContext, RMAppState.FINISHED, "appDiagnostics", 1234, null);
    store.updateApplicationState(dummyApp);
    ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.newInstance(dummyAppId, 6);
    ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.newInstance(dummyAttemptId, oldAttemptState.getMasterContainer(), oldAttemptState.getAppAttemptTokens(), oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 111, oldAttemptState.getFinishTime(), 0, 0, 0, 0);
    store.updateApplicationAttemptState(dummyAttempt);
    // let things settle down
    Thread.sleep(1000);
    store.close();
    // check updated application state.
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState newRMState = store.loadState();
    Map<ApplicationId, ApplicationStateData> newRMAppState = newRMState.getApplicationState();
    assertNotNull(newRMAppState.get(dummyApp.getApplicationSubmissionContext().getApplicationId()));
    ApplicationStateData updatedAppState = newRMAppState.get(appId1);
    assertEquals(appState.getApplicationSubmissionContext().getApplicationId(), updatedAppState.getApplicationSubmissionContext().getApplicationId());
    assertEquals(appState.getSubmitTime(), updatedAppState.getSubmitTime());
    assertEquals(appState.getStartTime(), updatedAppState.getStartTime());
    assertEquals(appState.getUser(), updatedAppState.getUser());
    // new app state fields
    assertEquals(RMAppState.FINISHED, updatedAppState.getState());
    assertEquals("appDiagnostics", updatedAppState.getDiagnostics());
    assertEquals(1234, updatedAppState.getFinishTime());
    // check updated attempt state
    assertNotNull(newRMAppState.get(dummyApp.getApplicationSubmissionContext().getApplicationId()).getAttempt(dummyAttemptId));
    ApplicationAttemptStateData updatedAttemptState = updatedAppState.getAttempt(newAttemptState.getAttemptId());
    assertEquals(oldAttemptState.getAttemptId(), updatedAttemptState.getAttemptId());
    assertEquals(containerId2, updatedAttemptState.getMasterContainer().getId());
    assertArrayEquals(clientTokenKey2.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    // new attempt state fields
    assertEquals(RMAppAttemptState.FINISHED, updatedAttemptState.getState());
    assertEquals("myTrackingUrl", updatedAttemptState.getFinalTrackingUrl());
    assertEquals("attemptDiagnostics", updatedAttemptState.getDiagnostics());
    assertEquals(100, updatedAttemptState.getAMContainerExitStatus());
    assertEquals(FinalApplicationStatus.SUCCEEDED, updatedAttemptState.getFinalApplicationStatus());
    // assert store is in expected state after everything is cleaned
    assertTrue(stateStoreHelper.isFinalStateValid());
    store.close();
}

Example 53 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method setup.

@Before
@SuppressWarnings("unchecked")
public void setup() {
    conf = new CapacitySchedulerConfiguration(new Configuration(false));
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 3000);
    // report "ideal" preempt
    conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, 1.0f);
    conf.setFloat(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, 1.0f);
    conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    // FairScheduler doesn't support this test,
    // Set CapacityScheduler as the scheduler for this test.
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    mClock = mock(Clock.class);
    mCS = mock(CapacityScheduler.class);
    when(mCS.getResourceCalculator()).thenReturn(rc);
    lm = mock(RMNodeLabelsManager.class);
    try {
        when(lm.isExclusiveNodeLabel(anyString())).thenReturn(true);
    } catch (IOException e) {
    // do nothing
    }
    when(mCS.getConfiguration()).thenReturn(conf);
    rmContext = mock(RMContext.class);
    when(mCS.getRMContext()).thenReturn(rmContext);
    when(mCS.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(rmContext.getNodeLabelManager()).thenReturn(lm);
    mDisp = mock(EventHandler.class);
    Dispatcher disp = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(disp);
    when(disp.getEventHandler()).thenReturn(mDisp);
    rand = new Random();
    long seed = rand.nextLong();
    System.out.println(name.getMethodName() + " SEED: " + seed);
    rand.setSeed(seed);
    appAlloc = 0;
}

Example 54 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method setup.

@SuppressWarnings("unchecked")
@Before
public void setup() {
    org.apache.log4j.Logger.getRootLogger().setLevel(org.apache.log4j.Level.DEBUG);
    conf = new CapacitySchedulerConfiguration(new Configuration(false));
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 3000);
    // report "ideal" preempt
    conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
    conf.setFloat(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, (float) 1.0);
    mClock = mock(Clock.class);
    cs = mock(CapacityScheduler.class);
    when(cs.getResourceCalculator()).thenReturn(rc);
    when(cs.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(cs.getConfiguration()).thenReturn(conf);
    nlm = mock(RMNodeLabelsManager.class);
    mDisp = mock(EventHandler.class);
    rmContext = mock(RMContext.class);
    when(rmContext.getNodeLabelManager()).thenReturn(nlm);
    Dispatcher disp = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(disp);
    when(disp.getEventHandler()).thenReturn(mDisp);
    when(cs.getRMContext()).thenReturn(rmContext);
    partitionToResource = new HashMap<>();
    nodeIdToSchedulerNodes = new HashMap<>();
    nameToCSQueues = new HashMap<>();
}

Example 55 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestNodesPage method testNodesBlockRenderForOpportunisticContainers.

@Test
public void testNodesBlockRenderForOpportunisticContainers() {
    final RMContext mockRMContext = TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack, 8 * TestRMWebApp.GiB);
    mockRMContext.getYarnConfiguration().setBoolean(YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
    injector = WebAppTests.createMockInjector(RMContext.class, mockRMContext, new Module() {

        @Override
        public void configure(Binder binder) {
            try {
                binder.bind(ResourceManager.class).toInstance(TestRMWebApp.mockRm(mockRMContext));
            } catch (IOException e) {
                throw new IllegalStateException(e);
            }
        }
    });
    injector.getInstance(NodesBlock.class).render();
    PrintWriter writer = injector.getInstance(PrintWriter.class);
    WebAppTests.flushOutput(injector);
    Mockito.verify(writer, Mockito.times(numberOfActualTableHeaders + numberOfThInMetricsTable + numberOfThForOpportunisticContainers)).print("<th");
    Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable)).print("<td");
}