Examples with RMAppAttempt org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt used on opensource projects

Example 91 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestCapacitySchedulerPlanFollower method setUp.

@Before
public void setUp() throws Exception {
    CapacityScheduler spyCs = new CapacityScheduler();
    cs = spy(spyCs);
    scheduler = cs;
    rmContext = TestUtils.getMockRMContext();
    spyRMContext = spy(rmContext);
    ConcurrentMap<ApplicationId, RMApp> spyApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>());
    RMApp rmApp = mock(RMApp.class);
    RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
    when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any())).thenReturn(rmAppAttempt);
    when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
    Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
    Mockito.doReturn(true).when(spyApps).containsKey((ApplicationId) Matchers.any());
    when(spyRMContext.getRMApps()).thenReturn(spyApps);
    when(spyRMContext.getScheduler()).thenReturn(scheduler);
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    ReservationSystemTestUtil.setupQueueConfiguration(csConf);
    cs.setConf(csConf);
    csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(csConf);
    when(csContext.getMinimumResourceCapability()).thenReturn(minAlloc);
    when(csContext.getMaximumResourceCapability()).thenReturn(maxAlloc);
    when(csContext.getClusterResource()).thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
    when(scheduler.getClusterResource()).thenReturn(Resources.createResource(125 * GB, 125));
    when(csContext.getResourceCalculator()).thenReturn(new DefaultResourceCalculator());
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(csConf);
    containerTokenSecretManager.rollMasterKey();
    when(csContext.getContainerTokenSecretManager()).thenReturn(containerTokenSecretManager);
    cs.setRMContext(spyRMContext);
    cs.init(csConf);
    cs.start();
    setupPlanFollower();
}

Example 92 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestRM method testAppOnMultiNode.

@Test(timeout = 30000)
public void testAppOnMultiNode() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    conf.set(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, "-1");
    MockRM rm = new MockRM(conf);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 5120);
    MockNM nm2 = rm.registerNode("h2:5678", 10240);
    RMApp app = rm.submitApp(2000);
    //kick the scheduling
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
    am.registerAppAttempt();
    //request for containers
    int request = 13;
    am.allocate("h1", 1000, request, new ArrayList<ContainerId>());
    //kick the scheduler
    List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    int contReceived = conts.size();
    while (contReceived < 3) {
        //only 3 containers are available on node1
        nm1.nodeHeartbeat(true);
        conts.addAll(am.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
        contReceived = conts.size();
        LOG.info("Got " + contReceived + " containers. Waiting to get " + 3);
        Thread.sleep(WAIT_SLEEP_MS);
    }
    Assert.assertEquals(3, conts.size());
    //send node2 heartbeat
    conts = am.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    contReceived = conts.size();
    while (contReceived < 10) {
        nm2.nodeHeartbeat(true);
        conts.addAll(am.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
        contReceived = conts.size();
        LOG.info("Got " + contReceived + " containers. Waiting to get " + 10);
        Thread.sleep(WAIT_SLEEP_MS);
    }
    Assert.assertEquals(10, conts.size());
    am.unregisterAppAttempt();
    nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    rm.stop();
}

Example 93 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestRM method testAppWithNoContainers.

@Test(timeout = 30000)
public void testAppWithNoContainers() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    MockRM rm = new MockRM(conf);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 5120);
    RMApp app = rm.submitApp(2000);
    //kick the scheduling
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
    am.registerAppAttempt();
    am.unregisterAppAttempt();
    nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    rm.stop();
}

Example 94 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestRM method testNMToken.

@Test(timeout = 40000)
public void testNMToken() throws Exception {
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        MockNM nm1 = rm.registerNode("h1:1234", 10000);
        NMTokenSecretManagerInRM nmTokenSecretManager = rm.getRMContext().getNMTokenSecretManager();
        // submitting new application
        RMApp app = rm.submitApp(1000);
        // start scheduling.
        nm1.nodeHeartbeat(true);
        // Starting application attempt and launching
        // It should get registered with NMTokenSecretManager.
        RMAppAttempt attempt = app.getCurrentAppAttempt();
        MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
        // This will register application master.
        am.registerAppAttempt();
        ArrayList<Container> containersReceivedForNM1 = new ArrayList<Container>();
        List<ContainerId> releaseContainerList = new ArrayList<ContainerId>();
        HashMap<String, Token> nmTokens = new HashMap<String, Token>();
        // initially requesting 2 containers.
        AllocateResponse response = am.allocate("h1", 1000, 2, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 2, nmTokens, nm1);
        Assert.assertEquals(1, nmTokens.size());
        // requesting 2 more containers.
        response = am.allocate("h1", 1000, 2, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 4, nmTokens, nm1);
        Assert.assertEquals(1, nmTokens.size());
        // We will be simulating NM restart so restarting newly added h2:1234
        // NM 2 now registers.
        MockNM nm2 = rm.registerNode("h2:1234", 10000);
        nm2.nodeHeartbeat(true);
        ArrayList<Container> containersReceivedForNM2 = new ArrayList<Container>();
        response = am.allocate("h2", 1000, 2, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 2, nmTokens, nm2);
        Assert.assertEquals(2, nmTokens.size());
        // Simulating NM-2 restart.
        nm2 = rm.registerNode("h2:1234", 10000);
        // Wait for reconnect to make it through the RM and create a new RMNode
        Map<NodeId, RMNode> nodes = rm.getRMContext().getRMNodes();
        while (nodes.get(nm2.getNodeId()).getLastNodeHeartBeatResponse().getResponseId() > 0) {
            Thread.sleep(WAIT_SLEEP_MS);
        }
        int interval = 40;
        // Wait for nm Token to be cleared.
        while (nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm2.getNodeId()) && interval-- > 0) {
            LOG.info("waiting for nmToken to be cleared for : " + nm2.getNodeId());
            Thread.sleep(WAIT_SLEEP_MS);
        }
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
        // removing NMToken for h2:1234
        nmTokens.remove(nm2.getNodeId().toString());
        Assert.assertEquals(1, nmTokens.size());
        // We should again receive the NMToken.
        response = am.allocate("h2", 1000, 2, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 4, nmTokens, nm2);
        Assert.assertEquals(2, nmTokens.size());
        // Now rolling over NMToken masterKey. it should resend the NMToken in
        // next allocate call.
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm1.getNodeId()));
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm2.getNodeId()));
        nmTokenSecretManager.rollMasterKey();
        nmTokenSecretManager.activateNextMasterKey();
        Assert.assertFalse(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm1.getNodeId()));
        Assert.assertFalse(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm2.getNodeId()));
        // It should not remove application attempt entry.
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
        nmTokens.clear();
        Assert.assertEquals(0, nmTokens.size());
        // We should again receive the NMToken.
        response = am.allocate("h2", 1000, 1, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 5, nmTokens, nm2);
        Assert.assertEquals(1, nmTokens.size());
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm2.getNodeId()));
        // After AM is finished making sure that nmtoken entry for app
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
        am.unregisterAppAttempt();
        // marking all the containers as finished.
        for (Container container : containersReceivedForNM1) {
            nm1.nodeHeartbeat(attempt.getAppAttemptId(), container.getId().getContainerId(), ContainerState.COMPLETE);
        }
        for (Container container : containersReceivedForNM2) {
            nm2.nodeHeartbeat(attempt.getAppAttemptId(), container.getId().getContainerId(), ContainerState.COMPLETE);
        }
        nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
        rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
        Assert.assertFalse(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
    } finally {
        rm.stop();
    }
}

Example 95 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class RMStateStoreTestBase method testRMAppStateStore.

void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, StoreStateVerifier verifier) throws Exception {
    long submitTime = System.currentTimeMillis();
    long startTime = System.currentTimeMillis() + 1234;
    Configuration conf = new YarnConfiguration();
    RMStateStore store = stateStoreHelper.getRMStateStore();
    TestDispatcher dispatcher = new TestDispatcher();
    store.setRMDispatcher(dispatcher);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getStateStore()).thenReturn(store);
    AMRMTokenSecretManager appTokenMgr = spy(new AMRMTokenSecretManager(conf, rmContext));
    MasterKeyData masterKeyData = appTokenMgr.createNewMasterKey();
    when(appTokenMgr.getMasterKey()).thenReturn(masterKeyData);
    ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
    ApplicationAttemptId attemptId1 = ApplicationAttemptId.fromString("appattempt_1352994193343_0001_000001");
    ApplicationId appId1 = attemptId1.getApplicationId();
    storeApp(store, appId1, submitTime, startTime);
    verifier.afterStoreApp(store, appId1);
    // create application token and client token key for attempt1
    Token<AMRMTokenIdentifier> appAttemptToken1 = generateAMRMToken(attemptId1, appTokenMgr);
    SecretKey clientTokenKey1 = clientToAMTokenMgr.createMasterKey(attemptId1);
    ContainerId containerId1 = storeAttempt(store, attemptId1, "container_1352994193343_0001_01_000001", appAttemptToken1, clientTokenKey1, dispatcher).getMasterContainer().getId();
    String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
    ApplicationAttemptId attemptId2 = ApplicationAttemptId.fromString(appAttemptIdStr2);
    // create application token and client token key for attempt2
    Token<AMRMTokenIdentifier> appAttemptToken2 = generateAMRMToken(attemptId2, appTokenMgr);
    SecretKey clientTokenKey2 = clientToAMTokenMgr.createMasterKey(attemptId2);
    ContainerId containerId2 = storeAttempt(store, attemptId2, "container_1352994193343_0001_02_000001", appAttemptToken2, clientTokenKey2, dispatcher).getMasterContainer().getId();
    ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString("appattempt_1352994193343_0002_000001");
    ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
    storeApp(store, appIdRemoved, submitTime, startTime);
    storeAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null, null, dispatcher);
    verifier.afterStoreAppAttempt(store, attemptIdRemoved);
    RMApp mockRemovedApp = mock(RMApp.class);
    RMAppAttemptMetrics mockRmAppAttemptMetrics = mock(RMAppAttemptMetrics.class);
    HashMap<ApplicationAttemptId, RMAppAttempt> attempts = new HashMap<ApplicationAttemptId, RMAppAttempt>();
    ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();
    context.setApplicationId(appIdRemoved);
    when(mockRemovedApp.getSubmitTime()).thenReturn(submitTime);
    when(mockRemovedApp.getApplicationSubmissionContext()).thenReturn(context);
    when(mockRemovedApp.getAppAttempts()).thenReturn(attempts);
    when(mockRemovedApp.getUser()).thenReturn("user1");
    RMAppAttempt mockRemovedAttempt = mock(RMAppAttempt.class);
    when(mockRemovedAttempt.getAppAttemptId()).thenReturn(attemptIdRemoved);
    when(mockRemovedAttempt.getRMAppAttemptMetrics()).thenReturn(mockRmAppAttemptMetrics);
    when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()).thenReturn(new AggregateAppResourceUsage(0, 0));
    attempts.put(attemptIdRemoved, mockRemovedAttempt);
    store.removeApplication(mockRemovedApp);
    // remove application directory recursively.
    storeApp(store, appIdRemoved, submitTime, startTime);
    storeAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null, null, dispatcher);
    store.removeApplication(mockRemovedApp);
    // let things settle down
    Thread.sleep(1000);
    store.close();
    // give tester a chance to modify app state in the store
    modifyAppState();
    // load state
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<ApplicationId, ApplicationStateData> rmAppState = state.getApplicationState();
    ApplicationStateData appState = rmAppState.get(appId1);
    // app is loaded
    assertNotNull(appState);
    // app is loaded correctly
    assertEquals(submitTime, appState.getSubmitTime());
    assertEquals(startTime, appState.getStartTime());
    // submission context is loaded correctly
    assertEquals(appId1, appState.getApplicationSubmissionContext().getApplicationId());
    ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
    // attempt1 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId1, attemptState.getAttemptId());
    assertEquals(-1000, attemptState.getAMContainerExitStatus());
    // attempt1 container is loaded correctly
    assertEquals(containerId1, attemptState.getMasterContainer().getId());
    // attempt1 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey1.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    assertEquals("context", appState.getCallerContext().getContext());
    attemptState = appState.getAttempt(attemptId2);
    // attempt2 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId2, attemptState.getAttemptId());
    // attempt2 container is loaded correctly
    assertEquals(containerId2, attemptState.getMasterContainer().getId());
    // attempt2 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey2.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    //******* update application/attempt state *******//
    ApplicationStateData appState2 = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), appState.getApplicationSubmissionContext(), RMAppState.FINISHED, "appDiagnostics", 1234, appState.getCallerContext());
    appState2.attempts.putAll(appState.attempts);
    store.updateApplicationState(appState2);
    ApplicationAttemptStateData oldAttemptState = attemptState;
    ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.newInstance(oldAttemptState.getAttemptId(), oldAttemptState.getMasterContainer(), oldAttemptState.getAppAttemptTokens(), oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 100, oldAttemptState.getFinishTime(), 0, 0, 0, 0);
    store.updateApplicationAttemptState(newAttemptState);
    // test updating the state of an app/attempt whose initial state was not
    // saved.
    ApplicationId dummyAppId = ApplicationId.newInstance(1234, 10);
    ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl();
    dummyContext.setApplicationId(dummyAppId);
    ApplicationStateData dummyApp = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), dummyContext, RMAppState.FINISHED, "appDiagnostics", 1234, null);
    store.updateApplicationState(dummyApp);
    ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.newInstance(dummyAppId, 6);
    ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.newInstance(dummyAttemptId, oldAttemptState.getMasterContainer(), oldAttemptState.getAppAttemptTokens(), oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 111, oldAttemptState.getFinishTime(), 0, 0, 0, 0);
    store.updateApplicationAttemptState(dummyAttempt);
    // let things settle down
    Thread.sleep(1000);
    store.close();
    // check updated application state.
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState newRMState = store.loadState();
    Map<ApplicationId, ApplicationStateData> newRMAppState = newRMState.getApplicationState();
    assertNotNull(newRMAppState.get(dummyApp.getApplicationSubmissionContext().getApplicationId()));
    ApplicationStateData updatedAppState = newRMAppState.get(appId1);
    assertEquals(appState.getApplicationSubmissionContext().getApplicationId(), updatedAppState.getApplicationSubmissionContext().getApplicationId());
    assertEquals(appState.getSubmitTime(), updatedAppState.getSubmitTime());
    assertEquals(appState.getStartTime(), updatedAppState.getStartTime());
    assertEquals(appState.getUser(), updatedAppState.getUser());
    // new app state fields
    assertEquals(RMAppState.FINISHED, updatedAppState.getState());
    assertEquals("appDiagnostics", updatedAppState.getDiagnostics());
    assertEquals(1234, updatedAppState.getFinishTime());
    // check updated attempt state
    assertNotNull(newRMAppState.get(dummyApp.getApplicationSubmissionContext().getApplicationId()).getAttempt(dummyAttemptId));
    ApplicationAttemptStateData updatedAttemptState = updatedAppState.getAttempt(newAttemptState.getAttemptId());
    assertEquals(oldAttemptState.getAttemptId(), updatedAttemptState.getAttemptId());
    assertEquals(containerId2, updatedAttemptState.getMasterContainer().getId());
    assertArrayEquals(clientTokenKey2.getEncoded(), attemptState.getAppAttemptTokens().getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
    // new attempt state fields
    assertEquals(RMAppAttemptState.FINISHED, updatedAttemptState.getState());
    assertEquals("myTrackingUrl", updatedAttemptState.getFinalTrackingUrl());
    assertEquals("attemptDiagnostics", updatedAttemptState.getDiagnostics());
    assertEquals(100, updatedAttemptState.getAMContainerExitStatus());
    assertEquals(FinalApplicationStatus.SUCCEEDED, updatedAttemptState.getFinalApplicationStatus());
    // assert store is in expected state after everything is cleaned
    assertTrue(stateStoreHelper.isFinalStateValid());
    store.close();
}