Examples with NodeLabelsProvider org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider used on opensource projects

Example 1 with NodeLabelsProvider

use of org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider in project hadoop by apache.

the class NodeManager method createNodeLabelsProvider.

protected NodeLabelsProvider createNodeLabelsProvider(Configuration conf) throws IOException {
    NodeLabelsProvider provider = null;
    String providerString = conf.get(YarnConfiguration.NM_NODE_LABELS_PROVIDER_CONFIG, null);
    if (providerString == null || providerString.trim().length() == 0) {
        // Seems like Distributed Node Labels configuration is not enabled
        return provider;
    }
    switch(providerString.trim().toLowerCase()) {
        case YarnConfiguration.CONFIG_NODE_LABELS_PROVIDER:
            provider = new ConfigurationNodeLabelsProvider();
            break;
        case YarnConfiguration.SCRIPT_NODE_LABELS_PROVIDER:
            provider = new ScriptBasedNodeLabelsProvider();
            break;
        default:
            try {
                Class<? extends NodeLabelsProvider> labelsProviderClass = conf.getClass(YarnConfiguration.NM_NODE_LABELS_PROVIDER_CONFIG, null, NodeLabelsProvider.class);
                provider = labelsProviderClass.newInstance();
            } catch (InstantiationException | IllegalAccessException | RuntimeException e) {
                LOG.error("Failed to create NodeLabelsProvider based on Configuration", e);
                throw new IOException("Failed to create NodeLabelsProvider : " + e.getMessage(), e);
            }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Distributed Node Labels is enabled" + " with provider class as : " + provider.getClass().toString());
    }
    return provider;
}

Example 2 with NodeLabelsProvider

use of org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider in project hadoop by apache.

the class TestNodeStatusUpdaterForLabels method testInvalidNodeLabelsFromProvider.

@Test(timeout = 20000)
public void testInvalidNodeLabelsFromProvider() throws InterruptedException, IOException {
    final ResourceTrackerForLabels resourceTracker = new ResourceTrackerForLabels();
    nm = new NodeManager() {

        @Override
        protected NodeLabelsProvider createNodeLabelsProvider(Configuration conf) throws IOException {
            return dummyLabelsProviderRef;
        }

        @Override
        protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker, NodeLabelsProvider labelsProvider) {
            return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics, labelsProvider) {

                @Override
                protected ResourceTracker getRMClient() {
                    return resourceTracker;
                }

                @Override
                protected void stopRMProxy() {
                    return;
                }
            };
        }
    };
    dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("P"));
    YarnConfiguration conf = createNMConfigForDistributeNodeLabels();
    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(8040, 10));
    nm.init(conf);
    resourceTracker.resetNMHeartbeatReceiveFlag();
    nm.start();
    // wait till the first heartbeat
    resourceTracker.waitTillHeartbeat();
    resourceTracker.resetNMHeartbeatReceiveFlag();
    // heartbeat with invalid labels
    dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("_.P"));
    sendOutofBandHeartBeat();
    resourceTracker.waitTillHeartbeat();
    assertNull("On Invalid Labels we need to retain earlier labels, HB " + "needs to send null", resourceTracker.labels);
    resourceTracker.resetNMHeartbeatReceiveFlag();
    // on next heartbeat same invalid labels will be given by the provider, but
    // again label validation check and reset RM with empty labels set should
    // not happen
    sendOutofBandHeartBeat();
    resourceTracker.waitTillHeartbeat();
    assertNull("NodeStatusUpdater need not send repeatedly empty labels on " + "invalid labels from provider ", resourceTracker.labels);
    resourceTracker.resetNMHeartbeatReceiveFlag();
}

Example 3 with NodeLabelsProvider

use of org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider in project hadoop by apache.

the class TestNodeManager method testCreationOfNodeLabelsProviderService.

@Test
public void testCreationOfNodeLabelsProviderService() throws InterruptedException {
    try {
        NodeManager nodeManager = new NodeManager();
        Configuration conf = new Configuration();
        NodeLabelsProvider labelsProviderService = nodeManager.createNodeLabelsProvider(conf);
        Assert.assertNull("LabelsProviderService should not be initialized in default configuration", labelsProviderService);
        // With valid className
        conf.set(YarnConfiguration.NM_NODE_LABELS_PROVIDER_CONFIG, "org.apache.hadoop.yarn.server.nodemanager.nodelabels.ConfigurationNodeLabelsProvider");
        labelsProviderService = nodeManager.createNodeLabelsProvider(conf);
        Assert.assertNotNull("LabelsProviderService should be initialized When " + "node labels provider class is configured", labelsProviderService);
        // With invalid className
        conf.set(YarnConfiguration.NM_NODE_LABELS_PROVIDER_CONFIG, "org.apache.hadoop.yarn.server.nodemanager.NodeManager");
        try {
            labelsProviderService = nodeManager.createNodeLabelsProvider(conf);
            Assert.fail("Expected to throw IOException on Invalid configuration");
        } catch (IOException e) {
        // exception expected on invalid configuration
        }
        Assert.assertNotNull("LabelsProviderService should be initialized When " + "node labels provider class is configured", labelsProviderService);
        // With valid whitelisted configurations
        conf.set(YarnConfiguration.NM_NODE_LABELS_PROVIDER_CONFIG, YarnConfiguration.CONFIG_NODE_LABELS_PROVIDER);
        labelsProviderService = nodeManager.createNodeLabelsProvider(conf);
        Assert.assertNotNull("LabelsProviderService should be initialized When " + "node labels provider class is configured", labelsProviderService);
    } catch (Exception e) {
        Assert.fail("Exception caught");
        e.printStackTrace();
    }
}

Example 4 with NodeLabelsProvider

use of org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider in project hadoop by apache.

the class TestNodeStatusUpdaterForLabels method testNodeStatusUpdaterForNodeLabels.

@Test(timeout = 20000)
public void testNodeStatusUpdaterForNodeLabels() throws InterruptedException, IOException {
    final ResourceTrackerForLabels resourceTracker = new ResourceTrackerForLabels();
    nm = new NodeManager() {

        @Override
        protected NodeLabelsProvider createNodeLabelsProvider(Configuration conf) throws IOException {
            return dummyLabelsProviderRef;
        }

        @Override
        protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker, NodeLabelsProvider labelsProvider) {
            return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics, labelsProvider) {

                @Override
                protected ResourceTracker getRMClient() {
                    return resourceTracker;
                }

                @Override
                protected void stopRMProxy() {
                    return;
                }
            };
        }
    };
    YarnConfiguration conf = createNMConfigForDistributeNodeLabels();
    conf.setLong(YarnConfiguration.NM_NODE_LABELS_RESYNC_INTERVAL, 2000);
    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(8040, 10));
    nm.init(conf);
    resourceTracker.resetNMHeartbeatReceiveFlag();
    nm.start();
    resourceTracker.waitTillRegister();
    assertNLCollectionEquals(dummyLabelsProviderRef.getNodeLabels(), resourceTracker.labels);
    // wait till the first heartbeat
    resourceTracker.waitTillHeartbeat();
    resourceTracker.resetNMHeartbeatReceiveFlag();
    // heartbeat with updated labels
    dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("P"));
    sendOutofBandHeartBeat();
    resourceTracker.waitTillHeartbeat();
    assertNLCollectionEquals(dummyLabelsProviderRef.getNodeLabels(), resourceTracker.labels);
    resourceTracker.resetNMHeartbeatReceiveFlag();
    // heartbeat without updating labels
    sendOutofBandHeartBeat();
    resourceTracker.waitTillHeartbeat();
    resourceTracker.resetNMHeartbeatReceiveFlag();
    assertNull("If no change in labels then null should be sent as part of request", resourceTracker.labels);
    // provider return with null labels
    dummyLabelsProviderRef.setNodeLabels(null);
    sendOutofBandHeartBeat();
    resourceTracker.waitTillHeartbeat();
    assertNotNull("If provider sends null then empty label set should be sent and not null", resourceTracker.labels);
    assertTrue("If provider sends null then empty labels should be sent", resourceTracker.labels.isEmpty());
    resourceTracker.resetNMHeartbeatReceiveFlag();
    // Since the resync interval is set to 2 sec in every alternate heartbeat
    // the labels will be send along with heartbeat.In loop we sleep for 1 sec
    // so that every sec 1 heartbeat is send.
    int nullLabels = 0;
    int nonNullLabels = 0;
    dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("P1"));
    for (int i = 0; i < 5; i++) {
        sendOutofBandHeartBeat();
        resourceTracker.waitTillHeartbeat();
        if (null == resourceTracker.labels) {
            nullLabels++;
        } else {
            Assert.assertEquals("In heartbeat PI labels should be send", toNodeLabelSet("P1"), resourceTracker.labels);
            nonNullLabels++;
        }
        resourceTracker.resetNMHeartbeatReceiveFlag();
        Thread.sleep(1000);
    }
    Assert.assertTrue("More than one heartbeat with empty labels expected", nullLabels > 1);
    Assert.assertTrue("More than one heartbeat with labels expected", nonNullLabels > 1);
    nm.stop();
}