Examples with CapacitySchedulerConfiguration org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration used on opensource projects

Example 6 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMAdminService method testRMHAWithFileSystemBasedConfiguration.

@Test
public void testRMHAWithFileSystemBasedConfiguration() throws IOException, YarnException {
    StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(HAServiceProtocol.RequestSource.REQUEST_BY_USER);
    updateConfigurationForRMHA();
    Configuration conf1 = new Configuration(configuration);
    conf1.set(YarnConfiguration.RM_HA_ID, "rm1");
    Configuration conf2 = new Configuration(configuration);
    conf2.set(YarnConfiguration.RM_HA_ID, "rm2");
    // upload default configurations
    uploadDefaultConfiguration();
    MockRM rm1 = null;
    MockRM rm2 = null;
    try {
        rm1 = new MockRM(conf1);
        rm1.init(conf1);
        rm1.start();
        Assert.assertTrue(rm1.getRMContext().getHAServiceState() == HAServiceState.STANDBY);
        rm2 = new MockRM(conf2);
        rm2.init(conf1);
        rm2.start();
        Assert.assertTrue(rm2.getRMContext().getHAServiceState() == HAServiceState.STANDBY);
        rm1.adminService.transitionToActive(requestInfo);
        Assert.assertTrue(rm1.getRMContext().getHAServiceState() == HAServiceState.ACTIVE);
        CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
        csConf.set(CapacitySchedulerConfiguration.MAXIMUM_SYSTEM_APPLICATIONS, "5000");
        uploadConfiguration(csConf, "capacity-scheduler.xml");
        rm1.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
        int maxApps = ((CapacityScheduler) rm1.getRMContext().getScheduler()).getConfiguration().getMaximumSystemApplications();
        Assert.assertEquals(maxApps, 5000);
        // Before failover happens, the maxApps is
        // still the default value on the standby rm : rm2
        int maxAppsBeforeFailOver = ((CapacityScheduler) rm2.getRMContext().getScheduler()).getConfiguration().getMaximumSystemApplications();
        Assert.assertEquals(maxAppsBeforeFailOver, 10000);
        // Do the failover
        rm1.adminService.transitionToStandby(requestInfo);
        rm2.adminService.transitionToActive(requestInfo);
        Assert.assertTrue(rm1.getRMContext().getHAServiceState() == HAServiceState.STANDBY);
        Assert.assertTrue(rm2.getRMContext().getHAServiceState() == HAServiceState.ACTIVE);
        int maxAppsAfter = ((CapacityScheduler) rm2.getRMContext().getScheduler()).getConfiguration().getMaximumSystemApplications();
        Assert.assertEquals(maxAppsAfter, 5000);
    } finally {
        if (rm1 != null) {
            rm1.stop();
        }
        if (rm2 != null) {
            rm2.stop();
        }
    }
}

Example 7 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMAdminService method testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider.

@Test
public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException {
    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
    //upload default configurations
    uploadDefaultConfiguration();
    try {
        rm = new MockRM(configuration);
        rm.init(configuration);
        rm.start();
    } catch (Exception ex) {
        fail("Should not get any exceptions");
    }
    CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
    int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications();
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    csConf.set(CapacitySchedulerConfiguration.MAXIMUM_SYSTEM_APPLICATIONS, "5000");
    uploadConfiguration(csConf, "capacity-scheduler.xml");
    rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
    int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications();
    Assert.assertEquals(maxAppsAfter, 5000);
    Assert.assertTrue(maxAppsAfter != maxAppsBefore);
}

Example 8 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestWorkPreservingRMRestart method testCapacityLeafQueueBecomesParentOnRecovery.

//Test behavior of an app if queue is changed from leaf to parent during
//recovery. Test case does following:
//1. Add an app to QueueB and start the attempt.
//2. Add 2 subqueues(QueueB1 and QueueB2) to QueueB, restart the RM, once with
//   fail fast config as false and once with fail fast as true.
//3. Verify that app was killed if fail fast is false.
//4. Verify that QueueException was thrown if fail fast is true.
@Test(timeout = 30000)
public void testCapacityLeafQueueBecomesParentOnRecovery() throws Exception {
    if (getSchedulerType() != SchedulerType.CAPACITY) {
        return;
    }
    conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
    conf.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class.getName());
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfiguration(csConf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(csConf);
    rm1 = new MockRM(csConf, memStore);
    rm1.start();
    MockNM nm = new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService());
    nm.registerNode();
    // Submit an app to QueueB.
    RMApp app = rm1.submitApp(1024, "app", USER_2, null, B);
    MockRM.launchAndRegisterAM(app, rm1, nm);
    assertEquals(rm1.getApplicationReport(app.getApplicationId()).getYarnApplicationState(), YarnApplicationState.RUNNING);
    // Take a copy of state store so that it can be reset to this state.
    RMState state = memStore.loadState();
    // Change scheduler config with child queues added to QueueB.
    csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfigurationChildOfB(csConf);
    String diags = "Application killed on recovery as it was submitted to " + "queue QueueB which is no longer a leaf queue after restart.";
    verifyAppRecoveryWithWrongQueueConfig(csConf, app, diags, memStore, state);
}

Example 9 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestWorkPreservingRMRestart method testRMRestartWithRemovedQueue.

// 1. submit an app to default queue and let it finish
// 2. restart rm with no default queue
// 3. getApplicationReport call should succeed (with no NPE)
@Test(timeout = 30000)
public void testRMRestartWithRemovedQueue() throws Exception {
    conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    conf.set(YarnConfiguration.YARN_ADMIN_ACL, "");
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    final RMApp app1 = rm1.submitApp(1024, "app1", USER_1, null);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { QUEUE_DOESNT_EXIST });
    final String noQueue = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_DOESNT_EXIST;
    csConf.setCapacity(noQueue, 100);
    rm2 = new MockRM(csConf, memStore);
    rm2.start();
    UserGroupInformation user2 = UserGroupInformation.createRemoteUser("user2");
    ApplicationReport report = user2.doAs(new PrivilegedExceptionAction<ApplicationReport>() {

        @Override
        public ApplicationReport run() throws Exception {
            return rm2.getApplicationReport(app1.getApplicationId());
        }
    });
    Assert.assertNotNull(report);
}

Example 10 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class ReservationSystemTestUtil method mockCapacityScheduler.

public CapacityScheduler mockCapacityScheduler(int numContainers) throws IOException {
    // stolen from TestCapacityScheduler
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(conf);
    CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
    cs.setConf(new YarnConfiguration());
    RMContext mockRmContext = createRMContext(conf);
    cs.setRMContext(mockRmContext);
    try {
        cs.serviceInit(conf);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    initializeRMContext(numContainers, cs, mockRmContext);
    return cs;
}