Examples with CapacitySchedulerConfiguration org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration used on opensource projects

Example 26 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestFairOrderingPolicy method testSizeBasedWeightNotAffectAppActivation.

@Test
public void testSizeBasedWeightNotAffectAppActivation() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    // Define top-level queues
    String queuePath = CapacitySchedulerConfiguration.ROOT + ".default";
    csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY);
    csConf.setOrderingPolicyParameter(queuePath, FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true");
    csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f);
    // inject node label manager
    MockRM rm = new MockRM(csConf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    // Get LeafQueue
    LeafQueue lq = (LeafQueue) cs.getQueue("default");
    OrderingPolicy<FiCaSchedulerApp> policy = lq.getOrderingPolicy();
    Assert.assertTrue(policy instanceof FairOrderingPolicy);
    Assert.assertTrue(((FairOrderingPolicy<FiCaSchedulerApp>) policy).getSizeBasedWeight());
    rm.registerNode("h1:1234", 10 * GB);
    // Submit 4 apps
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    Assert.assertEquals(1, lq.getNumActiveApplications());
    Assert.assertEquals(3, lq.getNumPendingApplications());
    // Try allocate once, #active-apps and #pending-apps should be still correct
    cs.handle(new NodeUpdateSchedulerEvent(rm.getRMContext().getRMNodes().get(NodeId.newInstance("h1", 1234))));
    Assert.assertEquals(1, lq.getNumActiveApplications());
    Assert.assertEquals(3, lq.getNumPendingApplications());
}

Example 27 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method setup.

@Before
@SuppressWarnings("unchecked")
public void setup() {
    conf = new CapacitySchedulerConfiguration(new Configuration(false));
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 3000);
    // report "ideal" preempt
    conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, 1.0f);
    conf.setFloat(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, 1.0f);
    conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    // FairScheduler doesn't support this test,
    // Set CapacityScheduler as the scheduler for this test.
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    mClock = mock(Clock.class);
    mCS = mock(CapacityScheduler.class);
    when(mCS.getResourceCalculator()).thenReturn(rc);
    lm = mock(RMNodeLabelsManager.class);
    try {
        when(lm.isExclusiveNodeLabel(anyString())).thenReturn(true);
    } catch (IOException e) {
    // do nothing
    }
    when(mCS.getConfiguration()).thenReturn(conf);
    rmContext = mock(RMContext.class);
    when(mCS.getRMContext()).thenReturn(rmContext);
    when(mCS.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(rmContext.getNodeLabelManager()).thenReturn(lm);
    mDisp = mock(EventHandler.class);
    Dispatcher disp = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(disp);
    when(disp.getEventHandler()).thenReturn(mDisp);
    rand = new Random();
    long seed = rand.nextLong();
    System.out.println(name.getMethodName() + " SEED: " + seed);
    rand.setSeed(seed);
    appAlloc = 0;
}

Example 28 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMNodeLabelsManager method getConfigurationWithQueueLabels.

private Configuration getConfigurationWithQueueLabels(Configuration config) {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config);
    // Define top-level queues
    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" });
    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
    conf.setCapacity(A, 100);
    conf.setAccessibleNodeLabels(A, ImmutableSet.of("x"));
    conf.setCapacityByLabel(A, "x", 100);
    return conf;
}

Example 29 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method setup.

@SuppressWarnings("unchecked")
@Before
public void setup() {
    org.apache.log4j.Logger.getRootLogger().setLevel(org.apache.log4j.Level.DEBUG);
    conf = new CapacitySchedulerConfiguration(new Configuration(false));
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 3000);
    // report "ideal" preempt
    conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
    conf.setFloat(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, (float) 1.0);
    mClock = mock(Clock.class);
    cs = mock(CapacityScheduler.class);
    when(cs.getResourceCalculator()).thenReturn(rc);
    when(cs.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(cs.getConfiguration()).thenReturn(conf);
    nlm = mock(RMNodeLabelsManager.class);
    mDisp = mock(EventHandler.class);
    rmContext = mock(RMContext.class);
    when(rmContext.getNodeLabelManager()).thenReturn(nlm);
    Dispatcher disp = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(disp);
    when(disp.getEventHandler()).thenReturn(mDisp);
    when(cs.getRMContext()).thenReturn(rmContext);
    partitionToResource = new HashMap<>();
    nodeIdToSchedulerNodes = new HashMap<>();
    nameToCSQueues = new HashMap<>();
}

Example 30 with CapacitySchedulerConfiguration

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration in project hadoop by apache.

the class TestRMWebApp method mockFifoScheduler.

public static FifoScheduler mockFifoScheduler(RMContext rmContext) throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupFifoQueueConfiguration(conf);
    FifoScheduler fs = new FifoScheduler();
    fs.setConf(new YarnConfiguration());
    fs.setRMContext(rmContext);
    fs.init(conf);
    return fs;
}