Examples with RMNodeLabelsManager org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager used on opensource projects

Example 16 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestNMReconnect method testCompareRMNodeAfterReconnect.

@Test
public void testCompareRMNodeAfterReconnect() throws Exception {
    AbstractYarnScheduler scheduler = getScheduler();
    Configuration yarnConf = new YarnConfiguration();
    ConfigurationProvider configurationProvider = ConfigurationProviderFactory.getConfigurationProvider(yarnConf);
    configurationProvider.init(yarnConf);
    context.setConfigurationProvider(configurationProvider);
    RMNodeLabelsManager nlm = new RMNodeLabelsManager();
    nlm.init(yarnConf);
    nlm.start();
    context.setNodeLabelManager(nlm);
    scheduler.setRMContext(context);
    scheduler.init(yarnConf);
    scheduler.start();
    dispatcher.register(SchedulerEventType.class, scheduler);
    String hostname1 = "localhost1";
    Resource capability = BuilderUtils.newResource(4096, 4);
    RegisterNodeManagerRequest request1 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
    NodeId nodeId1 = NodeId.newInstance(hostname1, 0);
    request1.setNodeId(nodeId1);
    request1.setHttpPort(0);
    request1.setResource(capability);
    resourceTrackerService.registerNodeManager(request1);
    Assert.assertNotNull(context.getRMNodes().get(nodeId1));
    // verify Scheduler and RMContext use same RMNode reference.
    Assert.assertTrue(scheduler.getSchedulerNode(nodeId1).getRMNode() == context.getRMNodes().get(nodeId1));
    Assert.assertEquals(context.getRMNodes().get(nodeId1).getTotalCapability(), capability);
    Resource capability1 = BuilderUtils.newResource(2048, 2);
    request1.setResource(capability1);
    resourceTrackerService.registerNodeManager(request1);
    Assert.assertNotNull(context.getRMNodes().get(nodeId1));
    // verify Scheduler and RMContext use same RMNode reference
    // after reconnect.
    Assert.assertTrue(scheduler.getSchedulerNode(nodeId1).getRMNode() == context.getRMNodes().get(nodeId1));
    // verify RMNode's capability is changed.
    Assert.assertEquals(context.getRMNodes().get(nodeId1).getTotalCapability(), capability1);
    nlm.stop();
    scheduler.stop();
}

Example 17 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestCapacityScheduler method testAppAttemptLocalityStatistics.

@Test
public void testAppAttemptLocalityStatistics() throws Exception {
    Configuration conf = TestUtils.getConfigurationWithMultipleQueues(new Configuration(false));
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    MockRM rm = new MockRM(conf) {

        protected RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.start();
    MockNM nm1 = new MockNM("h1:1234", 200 * GB, rm.getResourceTrackerService());
    nm1.registerNode();
    // Launch app1 in queue=a1
    RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a");
    // Got one offswitch request and offswitch allocation
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
    // am1 asks for 1 GB resource on h1/default-rack/offswitch
    am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(1 * GB), 2), ResourceRequest.newInstance(Priority.newInstance(1), "/default-rack", Resources.createResource(1 * GB), 2), ResourceRequest.newInstance(Priority.newInstance(1), "h1", Resources.createResource(1 * GB), 1)), null);
    CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
    // Got one nodelocal request and nodelocal allocation
    cs.nodeUpdate(rm.getRMContext().getRMNodes().get(nm1.getNodeId()));
    // Got one nodelocal request and racklocal allocation
    cs.nodeUpdate(rm.getRMContext().getRMNodes().get(nm1.getNodeId()));
    RMAppAttemptMetrics attemptMetrics = rm.getRMContext().getRMApps().get(app1.getApplicationId()).getCurrentAppAttempt().getRMAppAttemptMetrics();
    // We should get one node-local allocation, one rack-local allocation
    // And one off-switch allocation
    Assert.assertArrayEquals(new int[][] { { 1, 0, 0 }, { 0, 1, 0 }, { 0, 0, 1 } }, attemptMetrics.getLocalityStatistics());
}

Example 18 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestCapacitySchedulerAsyncScheduling method testAsyncContainerAllocation.

public void testAsyncContainerAllocation(int numThreads) throws Exception {
    conf.setInt(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD, numThreads);
    conf.setInt(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".scheduling-interval-ms", 100);
    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    // inject node label manager
    MockRM rm = new MockRM(TestUtils.getConfigurationWithMultipleQueues(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    List<MockNM> nms = new ArrayList<>();
    // Add 10 nodes to the cluster, in the cluster we have 200 GB resource
    for (int i = 0; i < 10; i++) {
        nms.add(rm.registerNode("h-" + i + ":1234", 20 * GB));
    }
    List<MockAM> ams = new ArrayList<>();
    // Add 3 applications to the cluster, one app in one queue
    // the i-th app ask (20 * i) containers. So in total we will have
    // 123G container allocated
    // 3 AMs
    int totalAsked = 3 * GB;
    for (int i = 0; i < 3; i++) {
        RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, Character.toString((char) (i % 34 + 97)), 1, null, null, false);
        MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm);
        am.registerAppAttempt();
        ams.add(am);
    }
    for (int i = 0; i < 3; i++) {
        ams.get(i).allocate("*", 1024, 20 * (i + 1), new ArrayList<>());
        totalAsked += 20 * (i + 1) * GB;
    }
    // Wait for at most 15000 ms
    // ms
    int waitTime = 15000;
    while (waitTime > 0) {
        if (rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB() == totalAsked) {
            break;
        }
        Thread.sleep(50);
        waitTime -= 50;
    }
    Assert.assertEquals(rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB(), totalAsked);
    // Wait for another 2 sec to make sure we will not allocate more than
    // required
    // ms
    waitTime = 2000;
    while (waitTime > 0) {
        Assert.assertEquals(rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB(), totalAsked);
        waitTime -= 50;
        Thread.sleep(50);
    }
    rm.close();
}

Example 19 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestApplicationLimits method testApplicationLimitSubmit.

@Test(timeout = 120000)
public void testApplicationLimitSubmit() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    // set node -> label
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z"));
    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
    // inject node label manager
    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 4096);
    MockNM nm2 = rm.registerNode("h2:1234", 4096);
    MockNM nm3 = rm.registerNode("h3:1234", 4096);
    // Submit application to queue c where the default partition capacity is
    // zero
    RMApp app1 = rm.submitApp(GB, "app", "user", null, "c", false);
    rm.drainEvents();
    rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app1.getState());
    rm.killApp(app1.getApplicationId());
    RMApp app2 = rm.submitApp(GB, "app", "user", null, "a1", false);
    rm.drainEvents();
    rm.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app2.getState());
    // Check second application is rejected and based on queue level max
    // application app is rejected
    RMApp app3 = rm.submitApp(GB, "app", "user", null, "a1", false);
    rm.drainEvents();
    rm.waitForState(app3.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app3.getState());
    assertEquals("org.apache.hadoop.security.AccessControlException: " + "Queue root.a.a1 already has 1 applications, cannot accept " + "submission of application: " + app3.getApplicationId(), app3.getDiagnostics().toString());
    // based on Global limit of queue usert application is rejected
    RMApp app11 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app11.getState());
    RMApp app12 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app12.getState());
    RMApp app13 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app13.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app13.getState());
    assertEquals("org.apache.hadoop.security.AccessControlException: Queue" + " root.d already has 2 applications from user user cannot" + " accept submission of application: " + app13.getApplicationId(), app13.getDiagnostics().toString());
    // based on system max limit application is rejected
    RMApp app14 = rm.submitApp(GB, "app", "user2", null, "a2", false);
    rm.drainEvents();
    rm.waitForState(app14.getApplicationId(), RMAppState.ACCEPTED);
    RMApp app15 = rm.submitApp(GB, "app", "user2", null, "a2", false);
    rm.drainEvents();
    rm.waitForState(app15.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app15.getState());
    assertEquals("Maximum system application limit reached,cannot" + " accept submission of application: " + app15.getApplicationId(), app15.getDiagnostics().toString());
    rm.killApp(app2.getApplicationId());
    rm.killApp(app11.getApplicationId());
    rm.killApp(app13.getApplicationId());
    rm.killApp(app14.getApplicationId());
    rm.stop();
}

Example 20 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestCapacityScheduler method testMoveAppViolateQueueState.

@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
    resourceManager = new ResourceManager() {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
            mgr.init(getConfig());
            return mgr;
        }
    };
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(csConf);
    StringBuilder qState = new StringBuilder();
    qState.append(CapacitySchedulerConfiguration.PREFIX).append(B).append(CapacitySchedulerConfiguration.DOT).append(CapacitySchedulerConfiguration.STATE);
    csConf.set(qState.toString(), QueueState.STOPPED.name());
    YarnConfiguration conf = new YarnConfiguration(csConf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    resourceManager.init(conf);
    resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
    resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
    ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
    mockContext = mock(RMContext.class);
    when(mockContext.getConfigurationProvider()).thenReturn(new LocalConfigurationProvider());
    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
    // Register node1
    String host_0 = "host_0";
    NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(6 * GB, 1));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit application_0
    Application application_0 = new Application("user_0", "a1", resourceManager);
    // app + app attempt event sent to scheduler
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    // allocate
    application_0.schedule();
    // task_0_0 allocated
    nodeUpdate(nm_0);
    // Get allocations from the scheduler
    // task_0_0
    application_0.schedule();
    checkApplicationResourceUsage(3 * GB, application_0);
    checkNodeResourceUsage(3 * GB, nm_0);
    // b2 queue contains 3GB consumption app,
    // add another 3GB will hit max capacity limit on queue b
    scheduler.moveApplication(application_0.getApplicationId(), "b1");
}