Examples with RMNodeLabelsManager org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager used on opensource projects

Example 36 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestRMAdminService method testModifyLabelsOnUnknownNodes.

@Test
public void testModifyLabelsOnUnknownNodes() throws IOException, YarnException {
    // create RM and set it's ACTIVE, and set distributed node label
    // configuration to true
    rm = new MockRM();
    ((RMContextImpl) rm.getRMContext()).setHAServiceState(HAServiceState.ACTIVE);
    Map<NodeId, RMNode> rmNodes = rm.getRMContext().getRMNodes();
    rmNodes.put(NodeId.newInstance("host1", 1111), new RMNodeImpl(null, rm.getRMContext(), "host1", 0, 0, null, null, null));
    rmNodes.put(NodeId.newInstance("host2", 2222), new RMNodeImpl(null, rm.getRMContext(), "host2", 0, 0, null, null, null));
    rmNodes.put(NodeId.newInstance("host3", 3333), new RMNodeImpl(null, rm.getRMContext(), "host3", 0, 0, null, null, null));
    Map<NodeId, RMNode> rmInactiveNodes = rm.getRMContext().getInactiveRMNodes();
    rmInactiveNodes.put(NodeId.newInstance("host4", 4444), new RMNodeImpl(null, rm.getRMContext(), "host4", 0, 0, null, null, null));
    RMNodeLabelsManager labelMgr = rm.rmContext.getNodeLabelManager();
    // by default, distributed configuration for node label is disabled, this
    // should pass
    labelMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
    // replace known node
    ReplaceLabelsOnNodeRequest request1 = ReplaceLabelsOnNodeRequest.newInstance(ImmutableMap.of(NodeId.newInstance("host1", 1111), (Set<String>) ImmutableSet.of("x")));
    request1.setFailOnUnknownNodes(true);
    try {
        rm.adminService.replaceLabelsOnNode(request1);
    } catch (Exception ex) {
        fail("should not fail on known node");
    }
    // replace known node with wildcard port
    ReplaceLabelsOnNodeRequest request2 = ReplaceLabelsOnNodeRequest.newInstance(ImmutableMap.of(NodeId.newInstance("host1", 0), (Set<String>) ImmutableSet.of("x")));
    request2.setFailOnUnknownNodes(true);
    try {
        rm.adminService.replaceLabelsOnNode(request2);
    } catch (Exception ex) {
        fail("should not fail on known node");
    }
    // replace unknown node
    ReplaceLabelsOnNodeRequest request3 = ReplaceLabelsOnNodeRequest.newInstance(ImmutableMap.of(NodeId.newInstance("host5", 0), (Set<String>) ImmutableSet.of("x")));
    request3.setFailOnUnknownNodes(true);
    try {
        rm.adminService.replaceLabelsOnNode(request3);
        fail("Should fail on unknown node");
    } catch (Exception ex) {
    }
    // replace known node but wrong port
    ReplaceLabelsOnNodeRequest request4 = ReplaceLabelsOnNodeRequest.newInstance(ImmutableMap.of(NodeId.newInstance("host2", 1111), (Set<String>) ImmutableSet.of("x")));
    request4.setFailOnUnknownNodes(true);
    try {
        rm.adminService.replaceLabelsOnNode(request4);
        fail("Should fail on node with wrong port");
    } catch (Exception ex) {
    }
    // replace non-exist node but not check
    ReplaceLabelsOnNodeRequest request5 = ReplaceLabelsOnNodeRequest.newInstance(ImmutableMap.of(NodeId.newInstance("host5", 0), (Set<String>) ImmutableSet.of("x")));
    request5.setFailOnUnknownNodes(false);
    try {
        rm.adminService.replaceLabelsOnNode(request5);
    } catch (Exception ex) {
        fail("Should not fail on unknown node when " + "fail-on-unkown-nodes is set false");
    }
    // replace on inactive node
    ReplaceLabelsOnNodeRequest request6 = ReplaceLabelsOnNodeRequest.newInstance(ImmutableMap.of(NodeId.newInstance("host4", 0), (Set<String>) ImmutableSet.of("x")));
    request6.setFailOnUnknownNodes(true);
    try {
        rm.adminService.replaceLabelsOnNode(request6);
    } catch (Exception ex) {
        fail("should not fail on inactive node");
    }
    rm.close();
}

Example 37 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestRMAdminService method testModifyLabelsOnNodesWithCentralizedConfigurationDisabled.

@Test(expected = YarnException.class)
public void testModifyLabelsOnNodesWithCentralizedConfigurationDisabled() throws IOException, YarnException {
    // create RM and set it's ACTIVE, and set distributed node label
    // configuration to true
    MockRM rm = new MockRM();
    rm.adminService.isCentralizedNodeLabelConfiguration = false;
    ((RMContextImpl) rm.getRMContext()).setHAServiceState(HAServiceState.ACTIVE);
    RMNodeLabelsManager labelMgr = rm.rmContext.getNodeLabelManager();
    // by default, distributed configuration for node label is disabled, this
    // should pass
    labelMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
    rm.adminService.replaceLabelsOnNode(ReplaceLabelsOnNodeRequest.newInstance(ImmutableMap.of(NodeId.newInstance("host", 0), (Set<String>) ImmutableSet.of("x"))));
    rm.close();
}

Example 38 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestClientRMService method testGetLabelsToNodes.

@Test
public void testGetLabelsToNodes() throws Exception {
    MockRM rm = new MockRM() {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.getRMContext().getRMDelegationTokenSecretManager());
        }

        ;
    };
    rm.start();
    NodeLabel labelX = NodeLabel.newInstance("x", false);
    NodeLabel labelY = NodeLabel.newInstance("y", false);
    NodeLabel labelZ = NodeLabel.newInstance("z", false);
    RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
    labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY, labelZ));
    NodeId node1A = NodeId.newInstance("host1", 1234);
    NodeId node1B = NodeId.newInstance("host1", 5678);
    NodeId node2A = NodeId.newInstance("host2", 1234);
    NodeId node3A = NodeId.newInstance("host3", 1234);
    NodeId node3B = NodeId.newInstance("host3", 5678);
    Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
    map.put(node1A, ImmutableSet.of("x"));
    map.put(node1B, ImmutableSet.of("z"));
    map.put(node2A, ImmutableSet.of("y"));
    map.put(node3A, ImmutableSet.of("y"));
    map.put(node3B, ImmutableSet.of("z"));
    labelsMgr.replaceLabelsOnNode(map);
    // Create a client.
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    ApplicationClientProtocol client = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
    // Get node labels collection
    GetClusterNodeLabelsResponse response = client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
    Assert.assertTrue(response.getNodeLabelList().containsAll(Arrays.asList(labelX, labelY, labelZ)));
    // Get labels to nodes mapping
    GetLabelsToNodesResponse response1 = client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance());
    Map<String, Set<NodeId>> labelsToNodes = response1.getLabelsToNodes();
    Assert.assertTrue(labelsToNodes.keySet().containsAll(Arrays.asList(labelX.getName(), labelY.getName(), labelZ.getName())));
    Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll(Arrays.asList(node1A)));
    Assert.assertTrue(labelsToNodes.get(labelY.getName()).containsAll(Arrays.asList(node2A, node3A)));
    Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll(Arrays.asList(node1B, node3B)));
    // Get labels to nodes mapping for specific labels
    Set<String> setlabels = new HashSet<String>(Arrays.asList(new String[] { "x", "z" }));
    GetLabelsToNodesResponse response2 = client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels));
    labelsToNodes = response2.getLabelsToNodes();
    Assert.assertTrue(labelsToNodes.keySet().containsAll(Arrays.asList(labelX.getName(), labelZ.getName())));
    Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll(Arrays.asList(node1A)));
    Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll(Arrays.asList(node1B, node3B)));
    Assert.assertEquals(labelsToNodes.get(labelY.getName()), null);
    rpc.stopProxy(client, conf);
    rm.close();
}

Example 39 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestQueueParsing method testQueueParsingWhenLabelsNotExist.

@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfigurationWithLabels(csConf);
    CapacityScheduler capacityScheduler = new CapacityScheduler();
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null);
    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
    nodeLabelsManager.init(conf);
    nodeLabelsManager.start();
    rmContext.setNodeLabelManager(nodeLabelsManager);
    capacityScheduler.setConf(csConf);
    capacityScheduler.setRMContext(rmContext);
    capacityScheduler.init(csConf);
    capacityScheduler.start();
    ServiceOperations.stopQuietly(capacityScheduler);
    ServiceOperations.stopQuietly(nodeLabelsManager);
}

Example 40 with RMNodeLabelsManager

use of org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager in project hadoop by apache.

the class TestQueueParsing method testRMStartWrongNodeCapacity.

@Test(timeout = 60000, expected = java.lang.IllegalArgumentException.class)
public void testRMStartWrongNodeCapacity() throws Exception {
    YarnConfiguration config = new YarnConfiguration();
    nodeLabelManager = new NullRMNodeLabelsManager();
    nodeLabelManager.init(config);
    config.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config);
    // Define top-level queues
    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" });
    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
    conf.setCapacity(A, 100);
    conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z"));
    conf.setCapacityByLabel(A, "x", 100);
    conf.setCapacityByLabel(A, "y", 100);
    conf.setCapacityByLabel(A, "z", 70);
    MockRM rm = null;
    try {
        rm = new MockRM(conf) {

            @Override
            public RMNodeLabelsManager createNodeLabelManager() {
                return nodeLabelManager;
            }
        };
    } finally {
        IOUtils.closeStream(rm);
    }
}