Examples with NMTokenSecretManagerInRM org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM used on opensource projects

Example 31 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestCapacityScheduler method testPreemptionDisabled.

@Test
public void testPreemptionDisabled() throws Exception {
    CapacityScheduler cs = new CapacityScheduler();
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null);
    setupQueueConfiguration(conf);
    cs.setConf(new YarnConfiguration());
    cs.setRMContext(resourceManager.getRMContext());
    cs.init(conf);
    cs.start();
    cs.reinitialize(conf, rmContext);
    CSQueue rootQueue = cs.getRootQueue();
    CSQueue queueB = findQueue(rootQueue, B);
    CSQueue queueB2 = findQueue(queueB, B2);
    // When preemption turned on for the whole system
    // (yarn.resourcemanager.scheduler.monitor.enable=true), and with no other 
    // preemption properties set, queue root.b.b2 should be preemptable.
    assertFalse("queue " + B2 + " should default to preemptable", queueB2.getPreemptionDisabled());
    // Disable preemption at the root queue level.
    // The preemption property should be inherited from root all the
    // way down so that root.b.b2 should NOT be preemptable.
    conf.setPreemptionDisabled(rootQueue.getQueuePath(), true);
    cs.reinitialize(conf, rmContext);
    assertTrue("queue " + B2 + " should have inherited non-preemptability from root", queueB2.getPreemptionDisabled());
    // Enable preemption for root (grandparent) but disable for root.b (parent).
    // root.b.b2 should inherit property from parent and NOT be preemptable
    conf.setPreemptionDisabled(rootQueue.getQueuePath(), false);
    conf.setPreemptionDisabled(queueB.getQueuePath(), true);
    cs.reinitialize(conf, rmContext);
    assertTrue("queue " + B2 + " should have inherited non-preemptability from parent", queueB2.getPreemptionDisabled());
    // When preemption is turned on for root.b.b2, it should be preemptable
    // even though preemption is disabled on root.b (parent).
    conf.setPreemptionDisabled(queueB2.getQueuePath(), false);
    cs.reinitialize(conf, rmContext);
    assertFalse("queue " + B2 + " should have been preemptable", queueB2.getPreemptionDisabled());
}

Example 32 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestCapacityScheduler method testRefreshQueuesWithNewQueue.

@Test
public void testRefreshQueuesWithNewQueue() throws Exception {
    CapacityScheduler cs = new CapacityScheduler();
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(conf);
    cs.setConf(new YarnConfiguration());
    cs.setRMContext(resourceManager.getRMContext());
    cs.init(conf);
    cs.start();
    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null));
    checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
    // Add a new queue b4
    String B4 = B + ".b4";
    float B4_CAPACITY = 10;
    B3_CAPACITY -= B4_CAPACITY;
    try {
        conf.setCapacity(A, 80f);
        conf.setCapacity(B, 20f);
        conf.setQueues(B, new String[] { "b1", "b2", "b3", "b4" });
        conf.setCapacity(B1, B1_CAPACITY);
        conf.setCapacity(B2, B2_CAPACITY);
        conf.setCapacity(B3, B3_CAPACITY);
        conf.setCapacity(B4, B4_CAPACITY);
        cs.reinitialize(conf, mockContext);
        checkQueueCapacities(cs, 80f, 20f);
        // Verify parent for B4
        CSQueue rootQueue = cs.getRootQueue();
        CSQueue queueB = findQueue(rootQueue, B);
        CSQueue queueB4 = findQueue(queueB, B4);
        assertEquals(queueB, queueB4.getParent());
    } finally {
        B3_CAPACITY += B4_CAPACITY;
        cs.stop();
    }
}

Example 33 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestFifoScheduler method testUpdateResourceOnNode.

@Test(timeout = 2000)
public void testUpdateResourceOnNode() throws Exception {
    AsyncDispatcher dispatcher = new InlineDispatcher();
    Configuration conf = new Configuration();
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.rollMasterKey();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    FifoScheduler scheduler = new FifoScheduler();
    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
    NullRMNodeLabelsManager nlm = new NullRMNodeLabelsManager();
    nlm.init(new Configuration());
    rmContext.setNodeLabelManager(nlm);
    scheduler.setRMContext(rmContext);
    ((RMContextImpl) rmContext).setScheduler(scheduler);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(new Configuration(), rmContext);
    RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 4), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
    scheduler.handle(nodeEvent1);
    assertEquals(scheduler.getNumClusterNodes(), 1);
    Resource newResource = Resources.createResource(1024, 4);
    NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(newResource, ResourceOption.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
    scheduler.handle(node0ResourceUpdate);
    // SchedulerNode's total resource and available resource are changed.
    assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).getTotalResource().getMemorySize());
    assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).getUnallocatedResource().getMemorySize(), 1024);
    QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
    Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
    int _appId = 1;
    int _appAttemptId = 1;
    ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId);
    createMockRMApp(appAttemptId, rmContext);
    AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1", "user1");
    scheduler.handle(appEvent);
    AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);
    int memory = 1024;
    int priority = 1;
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory, node0.getHostName(), priority, 1);
    ResourceRequest rackLocal = createResourceRequest(memory, node0.getRackName(), priority, 1);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority, 1);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    // Before the node update event, there are one local request
    Assert.assertEquals(1, nodeLocal.getNumContainers());
    NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
    // Now schedule.
    scheduler.handle(node0Update);
    // After the node update event, check no local request
    Assert.assertEquals(0, nodeLocal.getNumContainers());
    // Also check that one container was scheduled
    SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
    Assert.assertEquals(1, info.getLiveContainers().size());
    // And check the default Queue now is full.
    queueInfo = scheduler.getQueueInfo(null, false, false);
    Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}

Example 34 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestRM method testNMToken.

@Test(timeout = 40000)
public void testNMToken() throws Exception {
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        MockNM nm1 = rm.registerNode("h1:1234", 10000);
        NMTokenSecretManagerInRM nmTokenSecretManager = rm.getRMContext().getNMTokenSecretManager();
        // submitting new application
        RMApp app = rm.submitApp(1000);
        // start scheduling.
        nm1.nodeHeartbeat(true);
        // Starting application attempt and launching
        // It should get registered with NMTokenSecretManager.
        RMAppAttempt attempt = app.getCurrentAppAttempt();
        MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
        // This will register application master.
        am.registerAppAttempt();
        ArrayList<Container> containersReceivedForNM1 = new ArrayList<Container>();
        List<ContainerId> releaseContainerList = new ArrayList<ContainerId>();
        HashMap<String, Token> nmTokens = new HashMap<String, Token>();
        // initially requesting 2 containers.
        AllocateResponse response = am.allocate("h1", 1000, 2, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 2, nmTokens, nm1);
        Assert.assertEquals(1, nmTokens.size());
        // requesting 2 more containers.
        response = am.allocate("h1", 1000, 2, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 4, nmTokens, nm1);
        Assert.assertEquals(1, nmTokens.size());
        // We will be simulating NM restart so restarting newly added h2:1234
        // NM 2 now registers.
        MockNM nm2 = rm.registerNode("h2:1234", 10000);
        nm2.nodeHeartbeat(true);
        ArrayList<Container> containersReceivedForNM2 = new ArrayList<Container>();
        response = am.allocate("h2", 1000, 2, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 2, nmTokens, nm2);
        Assert.assertEquals(2, nmTokens.size());
        // Simulating NM-2 restart.
        nm2 = rm.registerNode("h2:1234", 10000);
        // Wait for reconnect to make it through the RM and create a new RMNode
        Map<NodeId, RMNode> nodes = rm.getRMContext().getRMNodes();
        while (nodes.get(nm2.getNodeId()).getLastNodeHeartBeatResponse().getResponseId() > 0) {
            Thread.sleep(WAIT_SLEEP_MS);
        }
        int interval = 40;
        // Wait for nm Token to be cleared.
        while (nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm2.getNodeId()) && interval-- > 0) {
            LOG.info("waiting for nmToken to be cleared for : " + nm2.getNodeId());
            Thread.sleep(WAIT_SLEEP_MS);
        }
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
        // removing NMToken for h2:1234
        nmTokens.remove(nm2.getNodeId().toString());
        Assert.assertEquals(1, nmTokens.size());
        // We should again receive the NMToken.
        response = am.allocate("h2", 1000, 2, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 4, nmTokens, nm2);
        Assert.assertEquals(2, nmTokens.size());
        // Now rolling over NMToken masterKey. it should resend the NMToken in
        // next allocate call.
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm1.getNodeId()));
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm2.getNodeId()));
        nmTokenSecretManager.rollMasterKey();
        nmTokenSecretManager.activateNextMasterKey();
        Assert.assertFalse(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm1.getNodeId()));
        Assert.assertFalse(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm2.getNodeId()));
        // It should not remove application attempt entry.
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
        nmTokens.clear();
        Assert.assertEquals(0, nmTokens.size());
        // We should again receive the NMToken.
        response = am.allocate("h2", 1000, 1, releaseContainerList);
        Assert.assertEquals(0, response.getAllocatedContainers().size());
        allocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 5, nmTokens, nm2);
        Assert.assertEquals(1, nmTokens.size());
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptNMTokenPresent(attempt.getAppAttemptId(), nm2.getNodeId()));
        // After AM is finished making sure that nmtoken entry for app
        Assert.assertTrue(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
        am.unregisterAppAttempt();
        // marking all the containers as finished.
        for (Container container : containersReceivedForNM1) {
            nm1.nodeHeartbeat(attempt.getAppAttemptId(), container.getId().getContainerId(), ContainerState.COMPLETE);
        }
        for (Container container : containersReceivedForNM2) {
            nm2.nodeHeartbeat(attempt.getAppAttemptId(), container.getId().getContainerId(), ContainerState.COMPLETE);
        }
        nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
        rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
        Assert.assertFalse(nmTokenSecretManager.isApplicationAttemptRegistered(attempt.getAppAttemptId()));
    } finally {
        rm.stop();
    }
}

Example 35 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestRMWebAppFairScheduler method mockFairScheduler.

private static FairScheduler mockFairScheduler() throws IOException {
    FairScheduler fs = new FairScheduler();
    FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
    fs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null));
    fs.init(conf);
    return fs;
}