Examples with YarnConfiguration org.apache.hadoop.yarn.conf.YarnConfiguration used on opensource projects

Example 56 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestAppManager method testStateStoreAppLimitLessThanMemoryAppLimit.

@Test
public void testStateStoreAppLimitLessThanMemoryAppLimit() {
    long now = System.currentTimeMillis();
    RMContext rmContext = mockRMContext(10, now - 20000);
    Configuration conf = new YarnConfiguration();
    int maxAppsInMemory = 8;
    int maxAppsInStateStore = 4;
    conf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, maxAppsInMemory);
    conf.setInt(YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, maxAppsInStateStore);
    TestRMAppManager appMonitor = new TestRMAppManager(rmContext, conf);
    addToCompletedApps(appMonitor, rmContext);
    Assert.assertEquals("Number of completed apps incorrect", 10, appMonitor.getCompletedAppsListSize());
    appMonitor.checkAppNumCompletedLimit();
    Assert.assertEquals("Number of apps incorrect after # completed check", maxAppsInMemory, rmContext.getRMApps().size());
    Assert.assertEquals("Number of completed apps incorrect after check", maxAppsInMemory, appMonitor.getCompletedAppsListSize());
    int numRemoveAppsFromStateStore = 10 - maxAppsInStateStore;
    verify(rmContext.getStateStore(), times(numRemoveAppsFromStateStore)).removeApplication(isA(RMApp.class));
    Assert.assertEquals(maxAppsInStateStore, appMonitor.getCompletedAppsInStateStore());
}

Example 57 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestAppManager method testQueueSubmitWithNoPermission.

@Test
public void testQueueSubmitWithNoPermission() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName());
    conf.set(PREFIX + "root.acl_submit_applications", " ");
    conf.set(PREFIX + "root.acl_administer_queue", " ");
    conf.set(PREFIX + "root.default.acl_submit_applications", " ");
    conf.set(PREFIX + "root.default.acl_administer_queue", " ");
    conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
    MockRM mockRM = new MockRM(conf);
    ClientRMService rmService = mockRM.getClientRMService();
    SubmitApplicationRequest req = Records.newRecord(SubmitApplicationRequest.class);
    ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class);
    sub.setApplicationId(appId);
    ResourceRequest resReg = ResourceRequest.newInstance(Priority.newInstance(0), ResourceRequest.ANY, Resource.newInstance(1024, 1), 1);
    sub.setAMContainerResourceRequest(resReg);
    req.setApplicationSubmissionContext(sub);
    sub.setAMContainerSpec(mock(ContainerLaunchContext.class));
    try {
        rmService.submitApplication(req);
    } catch (Exception e) {
        e.printStackTrace();
        if (e instanceof YarnException) {
            Assert.assertTrue(e.getCause() instanceof AccessControlException);
        } else {
            Assert.fail("Yarn exception is expected : " + e.getMessage());
        }
    } finally {
        mockRM.close();
    }
}

Example 58 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestAppManager method testRMAppRetireSomeDifferentStates.

@Test
public void testRMAppRetireSomeDifferentStates() throws Exception {
    long now = System.currentTimeMillis();
    // these parameters don't matter, override applications below
    RMContext rmContext = mockRMContext(10, now - 20000);
    Configuration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, 2);
    conf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 2);
    TestRMAppManager appMonitor = new TestRMAppManager(rmContext, conf);
    // clear out applications map
    rmContext.getRMApps().clear();
    Assert.assertEquals("map isn't empty", 0, rmContext.getRMApps().size());
    // 6 applications are in final state, 4 are not in final state.
    // / set with various finished states
    RMApp app = new MockRMApp(0, now - 20000, RMAppState.KILLED);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    app = new MockRMApp(1, now - 200000, RMAppState.FAILED);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    app = new MockRMApp(2, now - 30000, RMAppState.FINISHED);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    app = new MockRMApp(3, now - 20000, RMAppState.RUNNING);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    app = new MockRMApp(4, now - 20000, RMAppState.NEW);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    // make sure it doesn't expire these since still running
    app = new MockRMApp(5, now - 10001, RMAppState.KILLED);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    app = new MockRMApp(6, now - 30000, RMAppState.ACCEPTED);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    app = new MockRMApp(7, now - 20000, RMAppState.SUBMITTED);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    app = new MockRMApp(8, now - 10001, RMAppState.FAILED);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    app = new MockRMApp(9, now - 20000, RMAppState.FAILED);
    rmContext.getRMApps().put(app.getApplicationId(), app);
    Assert.assertEquals("Number of apps incorrect before", 10, rmContext.getRMApps().size());
    // add them to completed apps list
    addToCompletedApps(appMonitor, rmContext);
    // shouldn't  have to many apps
    appMonitor.checkAppNumCompletedLimit();
    Assert.assertEquals("Number of apps incorrect after # completed check", 6, rmContext.getRMApps().size());
    Assert.assertEquals("Number of completed apps incorrect after check", 2, appMonitor.getCompletedAppsListSize());
    // 6 applications in final state, 4 of them are removed
    verify(rmContext.getStateStore(), times(4)).removeApplication(isA(RMApp.class));
}

Example 59 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestApplicationCleanup method setup.

@Before
public void setup() throws UnknownHostException {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    conf = new YarnConfiguration();
    UserGroupInformation.setConfiguration(conf);
    conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
    conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    Assert.assertTrue(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS > 1);
}

Example 60 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestYarnClientProtocolProvider method testClusterGetDelegationToken.

@Test
public void testClusterGetDelegationToken() throws Exception {
    Configuration conf = new Configuration(false);
    Cluster cluster = null;
    try {
        conf = new Configuration();
        conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
        cluster = new Cluster(conf);
        YARNRunner yrunner = (YARNRunner) cluster.getClient();
        GetDelegationTokenResponse getDTResponse = recordFactory.newRecordInstance(GetDelegationTokenResponse.class);
        org.apache.hadoop.yarn.api.records.Token rmDTToken = recordFactory.newRecordInstance(org.apache.hadoop.yarn.api.records.Token.class);
        rmDTToken.setIdentifier(ByteBuffer.wrap(new byte[2]));
        rmDTToken.setKind("Testclusterkind");
        rmDTToken.setPassword(ByteBuffer.wrap("testcluster".getBytes()));
        rmDTToken.setService("0.0.0.0:8032");
        getDTResponse.setRMDelegationToken(rmDTToken);
        final ApplicationClientProtocol cRMProtocol = mock(ApplicationClientProtocol.class);
        when(cRMProtocol.getDelegationToken(any(GetDelegationTokenRequest.class))).thenReturn(getDTResponse);
        ResourceMgrDelegate rmgrDelegate = new ResourceMgrDelegate(new YarnConfiguration(conf)) {

            @Override
            protected void serviceStart() throws Exception {
                assertTrue(this.client instanceof YarnClientImpl);
                this.client = spy(this.client);
                doNothing().when(this.client).close();
                ((YarnClientImpl) this.client).setRMClient(cRMProtocol);
            }
        };
        yrunner.setResourceMgrDelegate(rmgrDelegate);
        Token t = cluster.getDelegationToken(new Text(" "));
        assertTrue("Token kind is instead " + t.getKind().toString(), "Testclusterkind".equals(t.getKind().toString()));
    } finally {
        if (cluster != null) {
            cluster.close();
        }
    }
}