Examples with NullRMNodeLabelsManager org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager used on opensource projects

Example 1 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestResourceTrackerService method testNodeRegistrationWithInvalidLabelsSyntax.

@Test
public void testNodeRegistrationWithInvalidLabelsSyntax() throws Exception {
    writeToHostsFile("host2");
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
    conf.set(YarnConfiguration.NODELABEL_CONFIGURATION_TYPE, YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);
    final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();
    rm = new MockRM(conf) {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            return nodeLabelsMgr;
        }
    };
    rm.start();
    try {
        nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("X", "Y", "Z"));
    } catch (IOException e) {
        Assert.fail("Caught Exception while intializing");
        e.printStackTrace();
    }
    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = NodeId.newInstance("host2", 1234);
    Resource capability = BuilderUtils.newResource(1024, 1);
    req.setResource(capability);
    req.setNodeId(nodeId);
    req.setHttpPort(1234);
    req.setNMVersion(YarnVersionInfo.getVersion());
    req.setNodeLabels(toNodeLabelSet("#Y"));
    RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
    Assert.assertEquals("On Invalid Node Labels action is expected to be normal", NodeAction.NORMAL, response.getNodeAction());
    Assert.assertNull(nodeLabelsMgr.getNodeLabels().get(nodeId));
    Assert.assertNotNull(response.getDiagnosticsMessage());
    Assert.assertFalse("Node Labels should not accepted by RM If Invalid", response.getAreNodeLabelsAcceptedByRM());
    if (rm != null) {
        rm.stop();
    }
}

Example 2 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestResourceTrackerService method testNodeRegistrationWithLabels.

@Test
public void testNodeRegistrationWithLabels() throws Exception {
    writeToHostsFile("host2");
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
    conf.set(YarnConfiguration.NODELABEL_CONFIGURATION_TYPE, YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);
    final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();
    rm = new MockRM(conf) {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            return nodeLabelsMgr;
        }
    };
    rm.start();
    try {
        nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C"));
    } catch (IOException e) {
        Assert.fail("Caught Exception while intializing");
        e.printStackTrace();
    }
    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest registerReq = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = NodeId.newInstance("host2", 1234);
    Resource capability = BuilderUtils.newResource(1024, 1);
    registerReq.setResource(capability);
    registerReq.setNodeId(nodeId);
    registerReq.setHttpPort(1234);
    registerReq.setNMVersion(YarnVersionInfo.getVersion());
    registerReq.setNodeLabels(toSet(NodeLabel.newInstance("A")));
    RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(registerReq);
    Assert.assertEquals("Action should be normal on valid Node Labels", NodeAction.NORMAL, response.getNodeAction());
    assertCollectionEquals(nodeLabelsMgr.getNodeLabels().get(nodeId), NodeLabelsUtils.convertToStringSet(registerReq.getNodeLabels()));
    Assert.assertTrue("Valid Node Labels were not accepted by RM", response.getAreNodeLabelsAcceptedByRM());
    rm.stop();
}

Example 3 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestWorkPreservingRMRestartForNodeLabel method testWorkPreservingRestartForNodeLabel.

@Test
public void testWorkPreservingRestartForNodeLabel() throws Exception {
    // This test is pretty much similar to testContainerAllocateWithLabel.
    // Difference is, this test doesn't specify label expression in ResourceRequest,
    // instead, it uses default queue label expression
    // set node -> label
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y")));
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    conf = TestUtils.getConfigurationWithDefaultQueueLabels(conf);
    // inject node label manager
    MockRM rm1 = new MockRM(conf, memStore) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm1.getRMContext().setNodeLabelManager(mgr);
    rm1.start();
    // label = x
    MockNM nm1 = rm1.registerNode("h1:1234", 8000);
    // label = y
    MockNM nm2 = rm1.registerNode("h2:1234", 8000);
    // label = <empty>
    MockNM nm3 = rm1.registerNode("h3:1234", 8000);
    ContainerId containerId;
    // launch an app to queue a1 (label = x), and check all container will
    // be allocated in h1
    RMApp app1 = rm1.submitApp(200, "app", "user", null, "a1");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    // request a container.
    am1.allocate("*", 1024, 1, new ArrayList<ContainerId>());
    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    Assert.assertTrue(rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED));
    checkRMContainerLabelExpression(ContainerId.newContainerId(am1.getApplicationAttemptId(), 1), rm1, "x");
    checkRMContainerLabelExpression(ContainerId.newContainerId(am1.getApplicationAttemptId(), 2), rm1, "x");
    // launch an app to queue b1 (label = y), and check all container will
    // be allocated in h2
    RMApp app2 = rm1.submitApp(200, "app", "user", null, "b1");
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2);
    // request a container.
    am2.allocate("*", 1024, 1, new ArrayList<ContainerId>());
    containerId = ContainerId.newContainerId(am2.getApplicationAttemptId(), 2);
    Assert.assertTrue(rm1.waitForState(nm2, containerId, RMContainerState.ALLOCATED));
    checkRMContainerLabelExpression(ContainerId.newContainerId(am2.getApplicationAttemptId(), 1), rm1, "y");
    checkRMContainerLabelExpression(ContainerId.newContainerId(am2.getApplicationAttemptId(), 2), rm1, "y");
    // launch an app to queue c1 (label = ""), and check all container will
    // be allocated in h3
    RMApp app3 = rm1.submitApp(200, "app", "user", null, "c1");
    MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm3);
    // request a container.
    am3.allocate("*", 1024, 1, new ArrayList<ContainerId>());
    containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 2);
    Assert.assertTrue(rm1.waitForState(nm3, containerId, RMContainerState.ALLOCATED));
    checkRMContainerLabelExpression(ContainerId.newContainerId(am3.getApplicationAttemptId(), 1), rm1, "");
    checkRMContainerLabelExpression(ContainerId.newContainerId(am3.getApplicationAttemptId(), 2), rm1, "");
    // Re-start RM
    mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y")));
    MockRM rm2 = new MockRM(conf, memStore) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    nm2.setResourceTrackerService(rm2.getResourceTrackerService());
    nm3.setResourceTrackerService(rm2.getResourceTrackerService());
    // recover app
    NMContainerStatus app1c1 = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING, "x");
    NMContainerStatus app1c2 = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING, "x");
    nm1.registerNode(Arrays.asList(app1c1, app1c2), null);
    waitForNumContainersToRecover(2, rm2, am1.getApplicationAttemptId());
    checkRMContainerLabelExpression(ContainerId.newContainerId(am1.getApplicationAttemptId(), 1), rm1, "x");
    checkRMContainerLabelExpression(ContainerId.newContainerId(am1.getApplicationAttemptId(), 2), rm1, "x");
    NMContainerStatus app2c1 = TestRMRestart.createNMContainerStatus(am2.getApplicationAttemptId(), 1, ContainerState.RUNNING, "y");
    NMContainerStatus app2c2 = TestRMRestart.createNMContainerStatus(am2.getApplicationAttemptId(), 2, ContainerState.RUNNING, "y");
    nm2.registerNode(Arrays.asList(app2c1, app2c2), null);
    waitForNumContainersToRecover(2, rm2, am2.getApplicationAttemptId());
    checkRMContainerLabelExpression(ContainerId.newContainerId(am2.getApplicationAttemptId(), 1), rm1, "y");
    checkRMContainerLabelExpression(ContainerId.newContainerId(am2.getApplicationAttemptId(), 2), rm1, "y");
    NMContainerStatus app3c1 = TestRMRestart.createNMContainerStatus(am3.getApplicationAttemptId(), 1, ContainerState.RUNNING, "");
    NMContainerStatus app3c2 = TestRMRestart.createNMContainerStatus(am3.getApplicationAttemptId(), 2, ContainerState.RUNNING, "");
    nm3.registerNode(Arrays.asList(app3c1, app3c2), null);
    waitForNumContainersToRecover(2, rm2, am3.getApplicationAttemptId());
    checkRMContainerLabelExpression(ContainerId.newContainerId(am3.getApplicationAttemptId(), 1), rm1, "");
    checkRMContainerLabelExpression(ContainerId.newContainerId(am3.getApplicationAttemptId(), 2), rm1, "");
    // Check recovered resource usage
    checkAppResourceUsage("x", app1.getApplicationId(), rm1, 2 * GB);
    checkAppResourceUsage("y", app2.getApplicationId(), rm1, 2 * GB);
    checkAppResourceUsage("", app3.getApplicationId(), rm1, 2 * GB);
    checkQueueResourceUsage("x", "a1", rm1, 2 * GB);
    checkQueueResourceUsage("y", "b1", rm1, 2 * GB);
    checkQueueResourceUsage("", "c1", rm1, 2 * GB);
    checkQueueResourceUsage("x", "a", rm1, 2 * GB);
    checkQueueResourceUsage("y", "b", rm1, 2 * GB);
    checkQueueResourceUsage("", "c", rm1, 2 * GB);
    checkQueueResourceUsage("x", "root", rm1, 2 * GB);
    checkQueueResourceUsage("y", "root", rm1, 2 * GB);
    checkQueueResourceUsage("", "root", rm1, 2 * GB);
    rm1.close();
    rm2.close();
}

Example 4 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestContainerAllocation method setUp.

@Before
public void setUp() throws Exception {
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
}

Example 5 with NullRMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager in project hadoop by apache.

the class TestContainerResizing method setUp.

@Before
public void setUp() throws Exception {
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
}