Examples with CapacitySchedulerConfiguration org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration used on opensource projects

Example 11 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method testObserveOnly.

@Test
public void testObserveOnly() {
    int[][] qData = new int[][] { // abs
    { 100, 40, 40, 20 }, // maxCap
    { 100, 100, 100, 100 }, // used
    { 100, 90, 10, 0 }, // pending
    { 80, 10, 20, 50 }, // reserved
    { 0, 0, 0, 0 }, // apps
    { 2, 1, 1, 0 }, // req granularity
    { -1, 1, 1, 0 }, // subqueues
    { 3, 0, 0, 0 } };
    conf.setBoolean(CapacitySchedulerConfiguration.PREEMPTION_OBSERVE_ONLY, true);
    when(mCS.getConfiguration()).thenReturn(new CapacitySchedulerConfiguration(conf));
    ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
    policy.editSchedule();
    // verify even severe imbalance not affected
    verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class));
}

Example 12 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestWorkPreservingRMRestart method testCapacitySchedulerQueueRemovedRecovery.

//Test behavior of an app if queue is removed during recovery. Test case does
//following:
//1. Add some apps to two queues, attempt to add an app to a non-existant
//   queue to verify that the new logic is not in effect during normal app
//   submission
//2. Remove one of the queues, restart the RM, once with fail fast config as
//   false and once with fail fast as true.
//3. Verify that app was killed if fail fast is false.
//4. Verify that QueueException was thrown if fail fast is true.
@Test(timeout = 30000)
public void testCapacitySchedulerQueueRemovedRecovery() throws Exception {
    if (getSchedulerType() != SchedulerType.CAPACITY) {
        return;
    }
    conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
    conf.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class.getName());
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfiguration(csConf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(csConf);
    rm1 = new MockRM(csConf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    MockNM nm2 = new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    nm2.registerNode();
    RMApp app1_1 = rm1.submitApp(1024, "app1_1", USER_1, null, A);
    MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
    RMApp app1_2 = rm1.submitApp(1024, "app1_2", USER_1, null, A);
    MockAM am1_2 = MockRM.launchAndRegisterAM(app1_2, rm1, nm2);
    RMApp app2 = rm1.submitApp(1024, "app2", USER_2, null, B);
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2);
    assertEquals(rm1.getApplicationReport(app2.getApplicationId()).getYarnApplicationState(), YarnApplicationState.RUNNING);
    //Submit an app with a non existant queue to make sure it does not
    //cause a fatal failure in the non-recovery case
    RMApp appNA = rm1.submitApp(1024, "app1_2", USER_1, null, QUEUE_DOESNT_EXIST, false);
    // clear queue metrics
    rm1.clearQueueMetrics(app1_1);
    rm1.clearQueueMetrics(app1_2);
    rm1.clearQueueMetrics(app2);
    // Take a copy of state store so that it can be reset to this state.
    RMState state = memStore.loadState();
    // Set new configuration with QueueB removed.
    csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfigurationOnlyA(csConf);
    String diags = "Application killed on recovery as it was submitted to " + "queue QueueB which no longer exists after restart.";
    verifyAppRecoveryWithWrongQueueConfig(csConf, app2, diags, memStore, state);
}

Example 13 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestReservationSystem method initializeCapacityScheduler.

private CapacityScheduler initializeCapacityScheduler() {
    // stolen from TestCapacityScheduler
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    ReservationSystemTestUtil.setupQueueConfiguration(conf);
    CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
    cs.setConf(conf);
    mockRMContext = ReservationSystemTestUtil.createRMContext(conf);
    cs.setRMContext(mockRMContext);
    try {
        cs.serviceInit(conf);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    ReservationSystemTestUtil.initializeRMContext(10, cs, mockRMContext);
    return cs;
}

Example 14 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestFifoScheduler method testReconnectedNode.

@Test(timeout = 50000)
public void testReconnectedNode() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    conf.setQueues("default", new String[] { "default" });
    conf.setCapacity("default", 100);
    FifoScheduler fs = new FifoScheduler();
    fs.init(conf);
    fs.start();
    // mock rmContext to avoid NPE.
    RMContext context = mock(RMContext.class);
    fs.reinitialize(conf, null);
    fs.setRMContext(context);
    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2, "127.0.0.3");
    fs.handle(new NodeAddedSchedulerEvent(n1));
    fs.handle(new NodeAddedSchedulerEvent(n2));
    fs.handle(new NodeUpdateSchedulerEvent(n1));
    Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB());
    // reconnect n1 with downgraded memory
    n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1, "127.0.0.2");
    fs.handle(new NodeRemovedSchedulerEvent(n1));
    fs.handle(new NodeAddedSchedulerEvent(n1));
    fs.handle(new NodeUpdateSchedulerEvent(n1));
    Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB());
    fs.stop();
}

Example 15 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMWebApp method mockCapacityScheduler.

public static CapacityScheduler mockCapacityScheduler() throws IOException {
    // stolen from TestCapacityScheduler
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(conf);
    CapacityScheduler cs = new CapacityScheduler();
    cs.setConf(new YarnConfiguration());
    RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null);
    rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
    cs.setRMContext(rmContext);
    cs.init(conf);
    return cs;
}