Examples with NullRMNodeLabelsManager org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager used on opensource projects

Example 11 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestRMWebApp method mockCapacityScheduler.

public static CapacityScheduler mockCapacityScheduler() throws IOException {
    // stolen from TestCapacityScheduler
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(conf);
    CapacityScheduler cs = new CapacityScheduler();
    cs.setConf(new YarnConfiguration());
    RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null);
    rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
    cs.setRMContext(rmContext);
    cs.init(conf);
    return cs;
}

Example 12 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestCapacityScheduler method testAppAttemptLocalityStatistics.

@Test
public void testAppAttemptLocalityStatistics() throws Exception {
    Configuration conf = TestUtils.getConfigurationWithMultipleQueues(new Configuration(false));
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    MockRM rm = new MockRM(conf) {

        protected RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.start();
    MockNM nm1 = new MockNM("h1:1234", 200 * GB, rm.getResourceTrackerService());
    nm1.registerNode();
    // Launch app1 in queue=a1
    RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a");
    // Got one offswitch request and offswitch allocation
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
    // am1 asks for 1 GB resource on h1/default-rack/offswitch
    am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(1 * GB), 2), ResourceRequest.newInstance(Priority.newInstance(1), "/default-rack", Resources.createResource(1 * GB), 2), ResourceRequest.newInstance(Priority.newInstance(1), "h1", Resources.createResource(1 * GB), 1)), null);
    CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
    // Got one nodelocal request and nodelocal allocation
    cs.nodeUpdate(rm.getRMContext().getRMNodes().get(nm1.getNodeId()));
    // Got one nodelocal request and racklocal allocation
    cs.nodeUpdate(rm.getRMContext().getRMNodes().get(nm1.getNodeId()));
    RMAppAttemptMetrics attemptMetrics = rm.getRMContext().getRMApps().get(app1.getApplicationId()).getCurrentAppAttempt().getRMAppAttemptMetrics();
    // We should get one node-local allocation, one rack-local allocation
    // And one off-switch allocation
    Assert.assertArrayEquals(new int[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } }, attemptMetrics.getLocalityStatistics());
}

Example 13 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestCapacitySchedulerAsyncScheduling method testAsyncContainerAllocation.

public void testAsyncContainerAllocation(int numThreads) throws Exception {
    conf.setInt(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD, numThreads);
    conf.setInt(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".scheduling-interval-ms", 100);
    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    // inject node label manager
    MockRM rm = new MockRM(TestUtils.getConfigurationWithMultipleQueues(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    List<MockNM> nms = new ArrayList<>();
    // Add 10 nodes to the cluster, in the cluster we have 200 GB resource
    for (int i = 0; i < 10; i++) {
        nms.add(rm.registerNode("h-" + i + ":1234", 20 * GB));
    }
    List<MockAM> ams = new ArrayList<>();
    // Add 3 applications to the cluster, one app in one queue
    // the i-th app ask (20 * i) containers. So in total we will have
    // 123G container allocated
    // 3 AMs
    int totalAsked = 3 * GB;
    for (int i = 0; i < 3; i++) {
        RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, Character.toString((char) (i % 34 + 97)), 1, null, null, false);
        MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm);
        am.registerAppAttempt();
        ams.add(am);
    }
    for (int i = 0; i < 3; i++) {
        ams.get(i).allocate("*", 1024, 20 * (i + 1), new ArrayList<>());
        totalAsked += 20 * (i + 1) * GB;
    }
    // Wait for at most 15000 ms
    // ms
    int waitTime = 15000;
    while (waitTime > 0) {
        if (rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB() == totalAsked) {
            break;
        }
        Thread.sleep(50);
        waitTime -= 50;
    }
    Assert.assertEquals(rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB(), totalAsked);
    // Wait for another 2 sec to make sure we will not allocate more than
    // required
    // ms
    waitTime = 2000;
    while (waitTime > 0) {
        Assert.assertEquals(rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB(), totalAsked);
        waitTime -= 50;
        Thread.sleep(50);
    }
    rm.close();
}

Example 14 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestApplicationLimits method testApplicationLimitSubmit.

@Test(timeout = 120000)
public void testApplicationLimitSubmit() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    // set node -> label
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z"));
    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
    // inject node label manager
    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 4096);
    MockNM nm2 = rm.registerNode("h2:1234", 4096);
    MockNM nm3 = rm.registerNode("h3:1234", 4096);
    // Submit application to queue c where the default partition capacity is
    // zero
    RMApp app1 = rm.submitApp(GB, "app", "user", null, "c", false);
    rm.drainEvents();
    rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app1.getState());
    rm.killApp(app1.getApplicationId());
    RMApp app2 = rm.submitApp(GB, "app", "user", null, "a1", false);
    rm.drainEvents();
    rm.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app2.getState());
    // Check second application is rejected and based on queue level max
    // application app is rejected
    RMApp app3 = rm.submitApp(GB, "app", "user", null, "a1", false);
    rm.drainEvents();
    rm.waitForState(app3.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app3.getState());
    assertEquals("org.apache.hadoop.security.AccessControlException: " + "Queue root.a.a1 already has 1 applications, cannot accept " + "submission of application: " + app3.getApplicationId(), app3.getDiagnostics().toString());
    // based on Global limit of queue usert application is rejected
    RMApp app11 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app11.getState());
    RMApp app12 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app12.getState());
    RMApp app13 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app13.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app13.getState());
    assertEquals("org.apache.hadoop.security.AccessControlException: Queue" + " root.d already has 2 applications from user user cannot" + " accept submission of application: " + app13.getApplicationId(), app13.getDiagnostics().toString());
    // based on system max limit application is rejected
    RMApp app14 = rm.submitApp(GB, "app", "user2", null, "a2", false);
    rm.drainEvents();
    rm.waitForState(app14.getApplicationId(), RMAppState.ACCEPTED);
    RMApp app15 = rm.submitApp(GB, "app", "user2", null, "a2", false);
    rm.drainEvents();
    rm.waitForState(app15.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app15.getState());
    assertEquals("Maximum system application limit reached,cannot" + " accept submission of application: " + app15.getApplicationId(), app15.getDiagnostics().toString());
    rm.killApp(app2.getApplicationId());
    rm.killApp(app11.getApplicationId());
    rm.killApp(app13.getApplicationId());
    rm.killApp(app14.getApplicationId());
    rm.stop();
}

Example 15 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestApplicationLimitsByPartition method setUp.

@Before
public void setUp() throws IOException {
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
}