Examples with PreemptionManager org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager used on opensource projects

Example 11 with PreemptionManager

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method setup.

@SuppressWarnings("unchecked")
@Before
public void setup() {
    org.apache.log4j.Logger.getRootLogger().setLevel(org.apache.log4j.Level.DEBUG);
    conf = new CapacitySchedulerConfiguration(new Configuration(false));
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 3000);
    // report "ideal" preempt
    conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
    conf.setFloat(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, (float) 1.0);
    mClock = mock(Clock.class);
    cs = mock(CapacityScheduler.class);
    when(cs.getResourceCalculator()).thenReturn(rc);
    when(cs.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(cs.getConfiguration()).thenReturn(conf);
    nlm = mock(RMNodeLabelsManager.class);
    mDisp = mock(EventHandler.class);
    rmContext = mock(RMContext.class);
    when(rmContext.getNodeLabelManager()).thenReturn(nlm);
    Dispatcher disp = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(disp);
    when(disp.getEventHandler()).thenReturn(mDisp);
    when(cs.getRMContext()).thenReturn(rmContext);
    partitionToResource = new HashMap<>();
    nodeIdToSchedulerNodes = new HashMap<>();
    nameToCSQueues = new HashMap<>();
}

Example 12 with PreemptionManager

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager in project hadoop by apache.

the class TestApplicationLimits method testLimitsComputation.

@Test
public void testLimitsComputation() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(csConf);
    YarnConfiguration conf = new YarnConfiguration();
    CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(conf);
    when(csContext.getMinimumResourceCapability()).thenReturn(Resources.createResource(GB, 1));
    when(csContext.getMaximumResourceCapability()).thenReturn(Resources.createResource(16 * GB, 16));
    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
    when(csContext.getRMContext()).thenReturn(rmContext);
    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
    // Say cluster has 100 nodes of 16G each
    Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16);
    when(csContext.getClusterResource()).thenReturn(clusterResource);
    Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
    CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook);
    LeafQueue queue = (LeafQueue) queues.get(A);
    LOG.info("Queue 'A' -" + " aMResourceLimit=" + queue.getAMResourceLimit() + " UserAMResourceLimit=" + queue.getUserAMResourceLimit());
    Resource amResourceLimit = Resource.newInstance(160 * GB, 1);
    assertEquals(queue.calculateAndGetAMResourceLimit(), amResourceLimit);
    assertEquals(queue.getUserAMResourceLimit(), Resource.newInstance(80 * GB, 1));
    // Assert in metrics
    assertEquals(queue.getMetrics().getAMResourceLimitMB(), amResourceLimit.getMemorySize());
    assertEquals(queue.getMetrics().getAMResourceLimitVCores(), amResourceLimit.getVirtualCores());
    assertEquals((int) (clusterResource.getMemorySize() * queue.getAbsoluteCapacity()), queue.getMetrics().getAvailableMB());
    // Add some nodes to the cluster & test new limits
    clusterResource = Resources.createResource(120 * 16 * GB);
    root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
    assertEquals(queue.calculateAndGetAMResourceLimit(), Resource.newInstance(192 * GB, 1));
    assertEquals(queue.getUserAMResourceLimit(), Resource.newInstance(96 * GB, 1));
    assertEquals((int) (clusterResource.getMemorySize() * queue.getAbsoluteCapacity()), queue.getMetrics().getAvailableMB());
    // should return -1 if per queue setting not set
    assertEquals((int) CapacitySchedulerConfiguration.UNDEFINED, csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
    int expectedMaxApps = (int) (CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * queue.getAbsoluteCapacity());
    assertEquals(expectedMaxApps, queue.getMaxApplications());
    int expectedMaxAppsPerUser = Math.min(expectedMaxApps, (int) (expectedMaxApps * (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()));
    assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
    // should default to global setting if per queue setting not set
    assertEquals((long) CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
    // Change the per-queue max AM resources percentage.
    csConf.setFloat(PREFIX + queue.getQueuePath() + ".maximum-am-resource-percent", 0.5f);
    // Re-create queues to get new configs.
    queues = new HashMap<String, CSQueue>();
    root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook);
    clusterResource = Resources.createResource(100 * 16 * GB);
    queue = (LeafQueue) queues.get(A);
    assertEquals((long) 0.5, (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
    assertEquals(queue.calculateAndGetAMResourceLimit(), Resource.newInstance(800 * GB, 1));
    assertEquals(queue.getUserAMResourceLimit(), Resource.newInstance(400 * GB, 1));
    // Change the per-queue max applications.
    csConf.setInt(PREFIX + queue.getQueuePath() + ".maximum-applications", 9999);
    // Re-create queues to get new configs.
    queues = new HashMap<String, CSQueue>();
    root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook);
    queue = (LeafQueue) queues.get(A);
    assertEquals(9999, (int) csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
    assertEquals(9999, queue.getMaxApplications());
    expectedMaxAppsPerUser = Math.min(9999, (int) (9999 * (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()));
    assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
}

Example 13 with PreemptionManager

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager in project hadoop by apache.

the class TestReservations method setup.

private void setup(CapacitySchedulerConfiguration csConf, boolean addUserLimits) throws Exception {
    csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
    final String newRoot = "root" + System.currentTimeMillis();
    // final String newRoot = "root";
    setupQueueConfiguration(csConf, newRoot, addUserLimits);
    YarnConfiguration conf = new YarnConfiguration();
    cs.setConf(conf);
    csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(conf);
    when(csContext.getMinimumResourceCapability()).thenReturn(Resources.createResource(GB, 1));
    when(csContext.getMaximumResourceCapability()).thenReturn(Resources.createResource(16 * GB, 12));
    when(csContext.getClusterResource()).thenReturn(Resources.createResource(100 * 16 * GB, 100 * 12));
    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(csContext.getRMContext()).thenReturn(rmContext);
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    when(csContext.getContainerTokenSecretManager()).thenReturn(containerTokenSecretManager);
    root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
    ResourceUsage queueResUsage = root.getQueueResourceUsage();
    when(csContext.getClusterResourceUsage()).thenReturn(queueResUsage);
    spyRMContext = spy(rmContext);
    when(spyRMContext.getScheduler()).thenReturn(cs);
    when(spyRMContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
    cs.setRMContext(spyRMContext);
    cs.init(csConf);
    cs.start();
    when(cs.getNumClusterNodes()).thenReturn(3);
}

Example 14 with PreemptionManager

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager in project hadoop by apache.

the class TestParentQueue method setUp.

@Before
public void setUp() throws Exception {
    rmContext = TestUtils.getMockRMContext();
    conf = new YarnConfiguration();
    csConf = new CapacitySchedulerConfiguration();
    csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConf()).thenReturn(conf);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getMinimumResourceCapability()).thenReturn(Resources.createResource(GB, 1));
    when(csContext.getMaximumResourceCapability()).thenReturn(Resources.createResource(16 * GB, 32));
    when(csContext.getClusterResource()).thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(csContext.getResourceCalculator()).thenReturn(resourceComparator);
    when(csContext.getRMContext()).thenReturn(rmContext);
}