Examples with CapacityScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler used on opensource projects

Example 31 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method setup.

@Before
@SuppressWarnings("unchecked")
public void setup() {
    conf = new CapacitySchedulerConfiguration(new Configuration(false));
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 3000);
    // report "ideal" preempt
    conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, 1.0f);
    conf.setFloat(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, 1.0f);
    conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    // FairScheduler doesn't support this test,
    // Set CapacityScheduler as the scheduler for this test.
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    mClock = mock(Clock.class);
    mCS = mock(CapacityScheduler.class);
    when(mCS.getResourceCalculator()).thenReturn(rc);
    lm = mock(RMNodeLabelsManager.class);
    try {
        when(lm.isExclusiveNodeLabel(anyString())).thenReturn(true);
    } catch (IOException e) {
    // do nothing
    }
    when(mCS.getConfiguration()).thenReturn(conf);
    rmContext = mock(RMContext.class);
    when(mCS.getRMContext()).thenReturn(rmContext);
    when(mCS.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(rmContext.getNodeLabelManager()).thenReturn(lm);
    mDisp = mock(EventHandler.class);
    Dispatcher disp = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(disp);
    when(disp.getEventHandler()).thenReturn(mDisp);
    rand = new Random();
    long seed = rand.nextLong();
    System.out.println(name.getMethodName() + " SEED: " + seed);
    rand.setSeed(seed);
    appAlloc = 0;
}

Example 32 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestWorkPreservingRMRestart method checkCSLeafQueue.

private void checkCSLeafQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) {
    LeafQueue leafQueue = (LeafQueue) app.getQueue();
    // assert queue used resources.
    assertEquals(usedResource, leafQueue.getUsedResources());
    assertEquals(numContainers, leafQueue.getNumContainers());
    ResourceCalculator calc = ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
    float usedCapacity = Resources.divide(calc, clusterResource, usedResource, queueResource);
    // assert queue used capacity
    assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
    float absoluteUsedCapacity = Resources.divide(calc, clusterResource, usedResource, clusterResource);
    // assert queue absolute capacity
    assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(), 1e-8);
    // assert user consumed resources.
    assertEquals(usedResource, leafQueue.getUser(app.getUser()).getUsed());
}

Example 33 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestRMWebServicesAppsModification method testUpdateAppPriority.

@Test(timeout = 90000)
public void testUpdateAppPriority() throws Exception {
    client().addFilter(new LoggingFilter(System.out));
    if (!(rm.getResourceScheduler() instanceof CapacityScheduler)) {
        // till the fair scheduler modifications for priority is completed
        return;
    }
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    Configuration conf = new Configuration();
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    cs.setClusterMaxPriority(conf);
    // default root queue allows anyone to have admin acl
    CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
    String[] queues = { "default", "test" };
    csconf.setQueues("root", queues);
    csconf.setCapacity("root.default", 50.0f);
    csconf.setCapacity("root.test", 50.0f);
    csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser");
    csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser");
    csconf.setAcl("root.test", QueueACL.ADMINISTER_QUEUE, "someuser");
    rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
    String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
    MediaType[] contentTypes = { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE };
    for (String mediaType : mediaTypes) {
        for (MediaType contentType : contentTypes) {
            RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
            amNodeManager.nodeHeartbeat(true);
            int modifiedPriority = 8;
            AppPriority priority = new AppPriority(modifiedPriority);
            Object entity;
            if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
                entity = appPriorityToJSON(priority);
            } else {
                entity = priority;
            }
            ClientResponse response = this.constructWebResource("apps", app.getApplicationId().toString(), "priority").entity(entity, contentType).accept(mediaType).put(ClientResponse.class);
            if (!isAuthenticationEnabled()) {
                assertResponseStatusCode(Status.UNAUTHORIZED, response.getStatusInfo());
                continue;
            }
            assertResponseStatusCode(Status.OK, response.getStatusInfo());
            if (mediaType.contains(MediaType.APPLICATION_JSON)) {
                verifyAppPriorityJson(response, modifiedPriority);
            } else {
                verifyAppPriorityXML(response, modifiedPriority);
            }
            response = this.constructWebResource("apps", app.getApplicationId().toString(), "priority").accept(mediaType).get(ClientResponse.class);
            assertResponseStatusCode(Status.OK, response.getStatusInfo());
            if (mediaType.contains(MediaType.APPLICATION_JSON)) {
                verifyAppPriorityJson(response, modifiedPriority);
            } else {
                verifyAppPriorityXML(response, modifiedPriority);
            }
            // check unauthorized
            app = rm.submitApp(CONTAINER_MB, "", "someuser");
            amNodeManager.nodeHeartbeat(true);
            response = this.constructWebResource("apps", app.getApplicationId().toString(), "priority").entity(entity, contentType).accept(mediaType).put(ClientResponse.class);
            assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
        }
    }
    rm.stop();
}

Example 34 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestRMWebServicesAppsModification method testAppMove.

@Test(timeout = 90000)
public void testAppMove() throws Exception {
    client().addFilter(new LoggingFilter(System.out));
    boolean isCapacityScheduler = rm.getResourceScheduler() instanceof CapacityScheduler;
    // default root queue allows anyone to have admin acl
    CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
    String[] queues = { "default", "test" };
    csconf.setQueues("root", queues);
    csconf.setCapacity("root.default", 50.0f);
    csconf.setCapacity("root.test", 50.0f);
    csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser");
    csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser");
    csconf.setAcl("root.test", QueueACL.ADMINISTER_QUEUE, "someuser");
    rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
    String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
    MediaType[] contentTypes = { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE };
    for (String mediaType : mediaTypes) {
        for (MediaType contentType : contentTypes) {
            RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
            amNodeManager.nodeHeartbeat(true);
            AppQueue targetQueue = new AppQueue("test");
            Object entity;
            if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
                entity = appQueueToJSON(targetQueue);
            } else {
                entity = targetQueue;
            }
            ClientResponse response = this.constructWebResource("apps", app.getApplicationId().toString(), "queue").entity(entity, contentType).accept(mediaType).put(ClientResponse.class);
            if (!isAuthenticationEnabled()) {
                assertResponseStatusCode(Status.UNAUTHORIZED, response.getStatusInfo());
                continue;
            }
            assertResponseStatusCode(Status.OK, response.getStatusInfo());
            String expectedQueue = "test";
            if (!isCapacityScheduler) {
                expectedQueue = "root.test";
            }
            if (mediaType.contains(MediaType.APPLICATION_JSON)) {
                verifyAppQueueJson(response, expectedQueue);
            } else {
                verifyAppQueueXML(response, expectedQueue);
            }
            Assert.assertEquals(expectedQueue, app.getQueue());
            // check unauthorized
            app = rm.submitApp(CONTAINER_MB, "", "someuser");
            amNodeManager.nodeHeartbeat(true);
            response = this.constructWebResource("apps", app.getApplicationId().toString(), "queue").entity(entity, contentType).accept(mediaType).put(ClientResponse.class);
            assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
            if (isCapacityScheduler) {
                Assert.assertEquals("default", app.getQueue());
            } else {
                Assert.assertEquals("root.someuser", app.getQueue());
            }
        }
    }
    rm.stop();
}

Example 35 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class RmController method scheduler.

public void scheduler() {
    // limit applications to those in states relevant to scheduling
    set(YarnWebParams.APP_STATE, StringHelper.cjoin(YarnApplicationState.NEW.toString(), YarnApplicationState.NEW_SAVING.toString(), YarnApplicationState.SUBMITTED.toString(), YarnApplicationState.ACCEPTED.toString(), YarnApplicationState.RUNNING.toString()));
    ResourceManager rm = getInstance(ResourceManager.class);
    ResourceScheduler rs = rm.getResourceScheduler();
    if (rs == null || rs instanceof CapacityScheduler) {
        setTitle("Capacity Scheduler");
        render(CapacitySchedulerPage.class);
        return;
    }
    if (rs instanceof FairScheduler) {
        setTitle("Fair Scheduler");
        render(FairSchedulerPage.class);
        return;
    }
    setTitle("Default Scheduler");
    render(DefaultSchedulerPage.class);
}