Examples with NMTokenSecretManagerInRM org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM used on opensource projects

Example 16 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestFifoScheduler method testNodeLocalAssignment.

@Test(timeout = 2000)
public void testNodeLocalAssignment() throws Exception {
    AsyncDispatcher dispatcher = new InlineDispatcher();
    Configuration conf = new Configuration();
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.rollMasterKey();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    FifoScheduler scheduler = new FifoScheduler();
    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
    scheduler.setRMContext(rmContext);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(new Configuration(), rmContext);
    RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(1024 * 64), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
    scheduler.handle(nodeEvent1);
    int _appId = 1;
    int _appAttemptId = 1;
    ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId);
    createMockRMApp(appAttemptId, rmContext);
    AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1", "user1");
    scheduler.handle(appEvent);
    AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);
    int memory = 64;
    int nConts = 3;
    int priority = 20;
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory, node0.getHostName(), priority, nConts);
    ResourceRequest rackLocal = createResourceRequest(memory, node0.getRackName(), priority, nConts);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority, nConts);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
    // Before the node update event, there are 3 local requests outstanding
    Assert.assertEquals(3, nodeLocal.getNumContainers());
    scheduler.handle(node0Update);
    // After the node update event, check that there are no more local requests
    // outstanding
    Assert.assertEquals(0, nodeLocal.getNumContainers());
    //Also check that the containers were scheduled
    SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
    Assert.assertEquals(3, info.getLiveContainers().size());
    scheduler.stop();
}

Example 17 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestRMWebApp method mockCapacityScheduler.

public static CapacityScheduler mockCapacityScheduler() throws IOException {
    // stolen from TestCapacityScheduler
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(conf);
    CapacityScheduler cs = new CapacityScheduler();
    cs.setConf(new YarnConfiguration());
    RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null);
    rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
    cs.setRMContext(rmContext);
    cs.init(conf);
    return cs;
}

Example 18 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestCapacityScheduler method testRefreshQueuesWithQueueDelete.

/**
   * Test for queue deletion.
   * @throws Exception
   */
@Test
public void testRefreshQueuesWithQueueDelete() throws Exception {
    CapacityScheduler cs = new CapacityScheduler();
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null);
    setupQueueConfiguration(conf);
    cs.setConf(new YarnConfiguration());
    cs.setRMContext(resourceManager.getRMContext());
    cs.init(conf);
    cs.start();
    cs.reinitialize(conf, rmContext);
    checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
    // test delete leaf queue when there is application running.
    Map<String, CSQueue> queues = cs.getCapacitySchedulerQueueManager().getQueues();
    String b1QTobeDeleted = "b1";
    LeafQueue csB1Queue = Mockito.spy((LeafQueue) queues.get(b1QTobeDeleted));
    when(csB1Queue.getState()).thenReturn(QueueState.DRAINING).thenReturn(QueueState.STOPPED);
    queues.put(b1QTobeDeleted, csB1Queue);
    conf = new CapacitySchedulerConfiguration();
    setupQueueConfigurationWithOutB1(conf);
    try {
        cs.reinitialize(conf, mockContext);
        fail("Expected to throw exception when refresh queue tries to delete a" + " queue with running apps");
    } catch (IOException e) {
    // ignore
    }
    // test delete leaf queue(root.b.b1) when there is no application running.
    conf = new CapacitySchedulerConfiguration();
    setupQueueConfigurationWithOutB1(conf);
    try {
        cs.reinitialize(conf, mockContext);
    } catch (IOException e) {
        fail("Expected to NOT throw exception when refresh queue tries to delete" + " a queue WITHOUT running apps");
    }
    CSQueue rootQueue = cs.getRootQueue();
    CSQueue queueB = findQueue(rootQueue, B);
    CSQueue queueB3 = findQueue(queueB, B1);
    assertNull("Refresh needs to support delete of leaf queue ", queueB3);
    // reset back to default configuration for testing parent queue delete
    conf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(conf);
    cs.reinitialize(conf, rmContext);
    checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
    // set the configurations such that it fails once but should be successfull
    // next time
    queues = cs.getCapacitySchedulerQueueManager().getQueues();
    CSQueue bQueue = Mockito.spy((ParentQueue) queues.get("b"));
    when(bQueue.getState()).thenReturn(QueueState.DRAINING).thenReturn(QueueState.STOPPED);
    queues.put("b", bQueue);
    bQueue = Mockito.spy((LeafQueue) queues.get("b1"));
    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
    queues.put("b1", bQueue);
    bQueue = Mockito.spy((LeafQueue) queues.get("b2"));
    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
    queues.put("b2", bQueue);
    bQueue = Mockito.spy((LeafQueue) queues.get("b3"));
    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
    queues.put("b3", bQueue);
    // test delete Parent queue when there is application running.
    conf = new CapacitySchedulerConfiguration();
    setupQueueConfigurationWithOutB(conf);
    try {
        cs.reinitialize(conf, mockContext);
        fail("Expected to throw exception when refresh queue tries to delete a" + " parent queue with running apps in children queue");
    } catch (IOException e) {
    // ignore
    }
    // test delete Parent queue when there is no application running.
    conf = new CapacitySchedulerConfiguration();
    setupQueueConfigurationWithOutB(conf);
    try {
        cs.reinitialize(conf, mockContext);
    } catch (IOException e) {
        fail("Expected to not throw exception when refresh queue tries to delete" + " a queue without running apps");
    }
    rootQueue = cs.getRootQueue();
    queueB = findQueue(rootQueue, B);
    String message = "Refresh needs to support delete of Parent queue and its children.";
    assertNull(message, queueB);
    assertNull(message, cs.getCapacitySchedulerQueueManager().getQueues().get("b"));
    assertNull(message, cs.getCapacitySchedulerQueueManager().getQueues().get("b1"));
    assertNull(message, cs.getCapacitySchedulerQueueManager().getQueues().get("b2"));
    cs.stop();
}

Example 19 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestQueueParsing method testQueueParsingWhenLabelsNotExist.

@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfigurationWithLabels(csConf);
    CapacityScheduler capacityScheduler = new CapacityScheduler();
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null);
    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
    nodeLabelsManager.init(conf);
    nodeLabelsManager.start();
    rmContext.setNodeLabelManager(nodeLabelsManager);
    capacityScheduler.setConf(csConf);
    capacityScheduler.setRMContext(rmContext);
    capacityScheduler.init(csConf);
    capacityScheduler.start();
    ServiceOperations.stopQuietly(capacityScheduler);
    ServiceOperations.stopQuietly(nodeLabelsManager);
}

Example 20 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestQueueParsing method testQueueOrderingPolicyUpdatedAfterReinitialize.

@Test
public void testQueueOrderingPolicyUpdatedAfterReinitialize() throws IOException {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfigurationWithoutLabels(csConf);
    YarnConfiguration conf = new YarnConfiguration(csConf);
    CapacityScheduler capacityScheduler = new CapacityScheduler();
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null);
    rmContext.setNodeLabelManager(nodeLabelManager);
    capacityScheduler.setConf(conf);
    capacityScheduler.setRMContext(rmContext);
    capacityScheduler.init(conf);
    capacityScheduler.start();
    // Add a new b4 queue
    csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".b", new String[] { "b1", "b2", "b3", "b4" });
    csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".b.b4", 0f);
    ParentQueue bQ = (ParentQueue) capacityScheduler.getQueue("b");
    checkEqualsToQueueSet(bQ.getChildQueues(), new String[] { "b1", "b2", "b3" });
    capacityScheduler.reinitialize(new YarnConfiguration(csConf), rmContext);
    // Check child queue of b
    checkEqualsToQueueSet(bQ.getChildQueues(), new String[] { "b1", "b2", "b3", "b4" });
    PriorityUtilizationQueueOrderingPolicy queueOrderingPolicy = (PriorityUtilizationQueueOrderingPolicy) bQ.getQueueOrderingPolicy();
    checkEqualsToQueueSet(queueOrderingPolicy.getQueues(), new String[] { "b1", "b2", "b3", "b4" });
    ServiceOperations.stopQuietly(capacityScheduler);
}