Examples with CapacityScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler used on opensource projects

Example 36 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestWorkPreservingRMRestart method checkCSLeafQueue.

private void checkCSLeafQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) {
    LeafQueue leafQueue = (LeafQueue) app.getQueue();
    // assert queue used resources.
    assertEquals(usedResource, leafQueue.getUsedResources());
    assertEquals(numContainers, leafQueue.getNumContainers());
    ResourceCalculator calc = ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
    float usedCapacity = Resources.divide(calc, clusterResource, usedResource, queueResource);
    // assert queue used capacity
    assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
    float absoluteUsedCapacity = Resources.divide(calc, clusterResource, usedResource, clusterResource);
    // assert queue absolute capacity
    assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(), 1e-8);
    // assert user consumed resources.
    assertEquals(usedResource, leafQueue.getUser(app.getUser()).getUsed());
}

Example 37 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestRMWebServicesAppsModification method testUpdateAppPriority.

@Test(timeout = 90000)
public void testUpdateAppPriority() throws Exception {
    client().addFilter(new LoggingFilter(System.out));
    if (!(rm.getResourceScheduler() instanceof CapacityScheduler)) {
        // till the fair scheduler modifications for priority is completed
        return;
    }
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    Configuration conf = new Configuration();
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    cs.setClusterMaxPriority(conf);
    // default root queue allows anyone to have admin acl
    CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
    String[] queues = { "default", "test" };
    csconf.setQueues("root", queues);
    csconf.setCapacity("root.default", 50.0f);
    csconf.setCapacity("root.test", 50.0f);
    csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser");
    csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser");
    csconf.setAcl("root.test", QueueACL.ADMINISTER_QUEUE, "someuser");
    rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
    String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
    MediaType[] contentTypes = { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE };
    for (String mediaType : mediaTypes) {
        for (MediaType contentType : contentTypes) {
            RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
            amNodeManager.nodeHeartbeat(true);
            int modifiedPriority = 8;
            AppPriority priority = new AppPriority(modifiedPriority);
            Object entity;
            if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
                entity = appPriorityToJSON(priority);
            } else {
                entity = priority;
            }
            ClientResponse response = this.constructWebResource("apps", app.getApplicationId().toString(), "priority").entity(entity, contentType).accept(mediaType).put(ClientResponse.class);
            if (!isAuthenticationEnabled()) {
                assertResponseStatusCode(Status.UNAUTHORIZED, response.getStatusInfo());
                continue;
            }
            assertResponseStatusCode(Status.OK, response.getStatusInfo());
            if (mediaType.contains(MediaType.APPLICATION_JSON)) {
                verifyAppPriorityJson(response, modifiedPriority);
            } else {
                verifyAppPriorityXML(response, modifiedPriority);
            }
            response = this.constructWebResource("apps", app.getApplicationId().toString(), "priority").accept(mediaType).get(ClientResponse.class);
            assertResponseStatusCode(Status.OK, response.getStatusInfo());
            if (mediaType.contains(MediaType.APPLICATION_JSON)) {
                verifyAppPriorityJson(response, modifiedPriority);
            } else {
                verifyAppPriorityXML(response, modifiedPriority);
            }
            // check unauthorized
            app = rm.submitApp(CONTAINER_MB, "", "someuser");
            amNodeManager.nodeHeartbeat(true);
            response = this.constructWebResource("apps", app.getApplicationId().toString(), "priority").entity(entity, contentType).accept(mediaType).put(ClientResponse.class);
            assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
        }
    }
    rm.stop();
}

Example 38 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestRMWebServicesAppsModification method testAppMove.

@Test(timeout = 90000)
public void testAppMove() throws Exception {
    client().addFilter(new LoggingFilter(System.out));
    boolean isCapacityScheduler = rm.getResourceScheduler() instanceof CapacityScheduler;
    // default root queue allows anyone to have admin acl
    CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
    String[] queues = { "default", "test" };
    csconf.setQueues("root", queues);
    csconf.setCapacity("root.default", 50.0f);
    csconf.setCapacity("root.test", 50.0f);
    csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser");
    csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser");
    csconf.setAcl("root.test", QueueACL.ADMINISTER_QUEUE, "someuser");
    rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
    String[] mediaTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
    MediaType[] contentTypes = { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE };
    for (String mediaType : mediaTypes) {
        for (MediaType contentType : contentTypes) {
            RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
            amNodeManager.nodeHeartbeat(true);
            AppQueue targetQueue = new AppQueue("test");
            Object entity;
            if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
                entity = appQueueToJSON(targetQueue);
            } else {
                entity = targetQueue;
            }
            ClientResponse response = this.constructWebResource("apps", app.getApplicationId().toString(), "queue").entity(entity, contentType).accept(mediaType).put(ClientResponse.class);
            if (!isAuthenticationEnabled()) {
                assertResponseStatusCode(Status.UNAUTHORIZED, response.getStatusInfo());
                continue;
            }
            assertResponseStatusCode(Status.OK, response.getStatusInfo());
            String expectedQueue = "test";
            if (!isCapacityScheduler) {
                expectedQueue = "root.test";
            }
            if (mediaType.contains(MediaType.APPLICATION_JSON)) {
                verifyAppQueueJson(response, expectedQueue);
            } else {
                verifyAppQueueXML(response, expectedQueue);
            }
            Assert.assertEquals(expectedQueue, app.getQueue());
            // check unauthorized
            app = rm.submitApp(CONTAINER_MB, "", "someuser");
            amNodeManager.nodeHeartbeat(true);
            response = this.constructWebResource("apps", app.getApplicationId().toString(), "queue").entity(entity, contentType).accept(mediaType).put(ClientResponse.class);
            assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
            if (isCapacityScheduler) {
                Assert.assertEquals("default", app.getQueue());
            } else {
                Assert.assertEquals("root.someuser", app.getQueue());
            }
        }
    }
    rm.stop();
}

Example 39 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicy method init.

public void init(Configuration config, RMContext context, PreemptableResourceScheduler sched) {
    LOG.info("Preemption monitor:" + this.getClass().getCanonicalName());
    assert null == scheduler : "Unexpected duplicate call to init";
    if (!(sched instanceof CapacityScheduler)) {
        throw new YarnRuntimeException("Class " + sched.getClass().getCanonicalName() + " not instance of " + CapacityScheduler.class.getCanonicalName());
    }
    rmContext = context;
    scheduler = (CapacityScheduler) sched;
    CapacitySchedulerConfiguration csConfig = scheduler.getConfiguration();
    maxIgnoredOverCapacity = csConfig.getDouble(CapacitySchedulerConfiguration.PREEMPTION_MAX_IGNORED_OVER_CAPACITY, CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_MAX_IGNORED_OVER_CAPACITY);
    naturalTerminationFactor = csConfig.getDouble(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_NATURAL_TERMINATION_FACTOR);
    maxWaitTime = csConfig.getLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_WAIT_TIME_BEFORE_KILL);
    monitoringInterval = csConfig.getLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_MONITORING_INTERVAL);
    percentageClusterPreemptionAllowed = csConfig.getFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, CapacitySchedulerConfiguration.DEFAULT_TOTAL_PREEMPTION_PER_ROUND);
    observeOnly = csConfig.getBoolean(CapacitySchedulerConfiguration.PREEMPTION_OBSERVE_ONLY, CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_OBSERVE_ONLY);
    lazyPreempionEnabled = csConfig.getBoolean(CapacitySchedulerConfiguration.LAZY_PREEMPTION_ENALBED, CapacitySchedulerConfiguration.DEFAULT_LAZY_PREEMPTION_ENABLED);
    maxAllowableLimitForIntraQueuePreemption = csConfig.getFloat(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT, CapacitySchedulerConfiguration.DEFAULT_INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT);
    minimumThresholdForIntraQueuePreemption = csConfig.getFloat(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_MINIMUM_THRESHOLD, CapacitySchedulerConfiguration.DEFAULT_INTRAQUEUE_PREEMPTION_MINIMUM_THRESHOLD);
    rc = scheduler.getResourceCalculator();
    nlm = scheduler.getRMContext().getNodeLabelManager();
    // Do we need white queue-priority preemption policy?
    boolean isQueuePriorityPreemptionEnabled = csConfig.getPUOrderingPolicyUnderUtilizedPreemptionEnabled();
    if (isQueuePriorityPreemptionEnabled) {
        candidatesSelectionPolicies.add(new QueuePriorityContainerCandidateSelector(this));
    }
    // Do we need to specially consider reserved containers?
    boolean selectCandidatesForResevedContainers = csConfig.getBoolean(CapacitySchedulerConfiguration.PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS, CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS);
    if (selectCandidatesForResevedContainers) {
        candidatesSelectionPolicies.add(new ReservedContainerCandidatesSelector(this));
    }
    // initialize candidates preemption selection policies
    candidatesSelectionPolicies.add(new FifoCandidatesSelector(this));
    // Do we need to specially consider intra queue
    boolean isIntraQueuePreemptionEnabled = csConfig.getBoolean(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ENABLED, CapacitySchedulerConfiguration.DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED);
    if (isIntraQueuePreemptionEnabled) {
        candidatesSelectionPolicies.add(new IntraQueueCandidatesSelector(this));
    }
}