Examples with FairScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler used on opensource projects

Example 11 with FairScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler in project hadoop by apache.

the class TestRMWebServicesFairScheduler method testClusterSchedulerWithSubQueues.

@Test
public void testClusterSchedulerWithSubQueues() throws JSONException, Exception {
    FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
    QueueManager queueManager = scheduler.getQueueManager();
    // create LeafQueue
    queueManager.getLeafQueue("root.q.subqueue1", true);
    queueManager.getLeafQueue("root.q.subqueue2", true);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("cluster").path("scheduler").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    JSONObject json = response.getEntity(JSONObject.class);
    JSONArray subQueueInfo = json.getJSONObject("scheduler").getJSONObject("schedulerInfo").getJSONObject("rootQueue").getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue");
    // subQueueInfo is consist of subqueue1 and subqueue2 info
    assertEquals(2, subQueueInfo.length());
    // Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
    try {
        subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
        fail("FairSchedulerQueueInfo should omit field 'childQueues'" + "if child queue is empty.");
    } catch (JSONException je) {
        assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
    }
}

Example 12 with FairScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler in project hadoop by apache.

the class ParameterizedSchedulerTestBase method configureScheduler.

@Before
public void configureScheduler() throws IOException, ClassNotFoundException {
    conf = new YarnConfiguration();
    Class schedulerClass = conf.getClass(YarnConfiguration.RM_SCHEDULER, Class.forName(YarnConfiguration.DEFAULT_RM_SCHEDULER));
    if (schedulerClass == FairScheduler.class) {
        schedulerType = SchedulerType.FAIR;
        configureFairScheduler(conf);
        scheduler = new FairScheduler();
    } else if (schedulerClass == CapacityScheduler.class) {
        schedulerType = SchedulerType.CAPACITY;
        scheduler = new CapacityScheduler();
        ((CapacityScheduler) scheduler).setConf(conf);
    }
}

Example 13 with FairScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler in project hadoop by apache.

the class TestRMWebAppFairScheduler method mockFairSchedulerWithoutApps.

private static FairScheduler mockFairSchedulerWithoutApps(RMContext rmContext) throws IOException {
    FairScheduler fs = new FairScheduler() {

        @Override
        public FSAppAttempt getSchedulerApp(ApplicationAttemptId applicationAttemptId) {
            return null;
        }

        @Override
        public FSAppAttempt getApplicationAttempt(ApplicationAttemptId applicationAttemptId) {
            return null;
        }
    };
    FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
    fs.setRMContext(rmContext);
    fs.init(conf);
    return fs;
}

Example 14 with FairScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler in project hadoop by apache.

the class TestRMWebAppFairScheduler method mockFairScheduler.

private static FairScheduler mockFairScheduler() throws IOException {
    FairScheduler fs = new FairScheduler();
    FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
    fs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null));
    fs.init(conf);
    return fs;
}

Example 15 with FairScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler in project hadoop by apache.

the class FairSchedulerMetrics method trackQueue.

@Override
public void trackQueue(String queueName) {
    trackedQueues.add(queueName);
    FairScheduler fair = (FairScheduler) scheduler;
    final FSQueue queue = fair.getQueueManager().getQueue(queueName);
    metrics.register("variable.queue." + queueName + ".demand.memory", new Gauge<Long>() {

        @Override
        public Long getValue() {
            return queue.getDemand().getMemorySize();
        }
    });
    metrics.register("variable.queue." + queueName + ".demand.vcores", new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return queue.getDemand().getVirtualCores();
        }
    });
    metrics.register("variable.queue." + queueName + ".usage.memory", new Gauge<Long>() {

        @Override
        public Long getValue() {
            return queue.getResourceUsage().getMemorySize();
        }
    });
    metrics.register("variable.queue." + queueName + ".usage.vcores", new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return queue.getResourceUsage().getVirtualCores();
        }
    });
    metrics.register("variable.queue." + queueName + ".minshare.memory", new Gauge<Long>() {

        @Override
        public Long getValue() {
            return queue.getMinShare().getMemorySize();
        }
    });
    metrics.register("variable.queue." + queueName + ".minshare.vcores", new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return queue.getMinShare().getVirtualCores();
        }
    });
    metrics.register("variable.queue." + queueName + ".maxshare.memory", new Gauge<Long>() {

        @Override
        public Long getValue() {
            if (!maxReset && SLSRunner.simulateInfoMap.containsKey("Number of nodes") && SLSRunner.simulateInfoMap.containsKey("Node memory (MB)") && SLSRunner.simulateInfoMap.containsKey("Node VCores")) {
                int numNMs = Integer.parseInt(SLSRunner.simulateInfoMap.get("Number of nodes").toString());
                int numMemoryMB = Integer.parseInt(SLSRunner.simulateInfoMap.get("Node memory (MB)").toString());
                int numVCores = Integer.parseInt(SLSRunner.simulateInfoMap.get("Node VCores").toString());
                totalMemoryMB = numNMs * numMemoryMB;
                totalVCores = numNMs * numVCores;
                maxReset = false;
            }
            return Math.min(queue.getMaxShare().getMemorySize(), totalMemoryMB);
        }
    });
    metrics.register("variable.queue." + queueName + ".maxshare.vcores", new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return Math.min(queue.getMaxShare().getVirtualCores(), totalVCores);
        }
    });
    metrics.register("variable.queue." + queueName + ".fairshare.memory", new Gauge<Long>() {

        @Override
        public Long getValue() {
            return queue.getFairShare().getMemorySize();
        }
    });
    metrics.register("variable.queue." + queueName + ".fairshare.vcores", new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return queue.getFairShare().getVirtualCores();
        }
    });
}