Examples with RMContext org.apache.hadoop.yarn.server.resourcemanager.RMContext used on opensource projects

Example 71 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestReservations method testGetAppToUnreserve.

@Test
public void testGetAppToUnreserve() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setup(csConf);
    final String user_0 = "user_0";
    final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 0);
    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
    FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, mock(ActiveUsersManager.class), spyRMContext);
    String host_0 = "host_0";
    FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 8 * GB);
    String host_1 = "host_1";
    FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, 8 * GB);
    Resource clusterResource = Resources.createResource(2 * 8 * GB);
    // Setup resource-requests
    Priority p = TestUtils.createMockPriority(5);
    SchedulerRequestKey priorityMap = toSchedulerKey(p);
    Resource capability = Resources.createResource(2 * GB, 0);
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
    RMContext rmContext = mock(RMContext.class);
    ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);
    DrainDispatcher drainDispatcher = new DrainDispatcher();
    when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
    when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
    when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
    when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
    when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(app_0.getApplicationId(), 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
    Container container = TestUtils.getMockContainer(containerId, node_1.getNodeID(), Resources.createResource(2 * GB), priorityMap.getPriority());
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttemptId, node_1.getNodeID(), "user", rmContext);
    Container container_1 = TestUtils.getMockContainer(containerId, node_0.getNodeID(), Resources.createResource(1 * GB), priorityMap.getPriority());
    RMContainer rmContainer_1 = new RMContainerImpl(container_1, SchedulerRequestKey.extractFrom(container_1), appAttemptId, node_0.getNodeID(), "user", rmContext);
    // no reserved containers
    NodeId unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, cs.getResourceCalculator(), clusterResource);
    assertEquals(null, unreserveId);
    // no reserved containers - reserve then unreserve
    app_0.reserve(node_0, priorityMap, rmContainer_1, container_1);
    app_0.unreserve(priorityMap, node_0, rmContainer_1);
    unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, cs.getResourceCalculator(), clusterResource);
    assertEquals(null, unreserveId);
    // no container large enough is reserved
    app_0.reserve(node_0, priorityMap, rmContainer_1, container_1);
    unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, cs.getResourceCalculator(), clusterResource);
    assertEquals(null, unreserveId);
    // reserve one that is now large enough
    app_0.reserve(node_1, priorityMap, rmContainer, container);
    unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, cs.getResourceCalculator(), clusterResource);
    assertEquals(node_1.getNodeID(), unreserveId);
}

Example 72 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestQueueState method testQueueStateTransit.

@Test(timeout = 15000)
public void testQueueStateTransit() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { Q1 });
    csConf.setQueues(Q1_PATH, new String[] { Q2, Q3 });
    csConf.setCapacity(Q1_PATH, 100);
    csConf.setCapacity(Q2_PATH, 50);
    csConf.setCapacity(Q3_PATH, 50);
    conf = new YarnConfiguration(csConf);
    cs = new CapacityScheduler();
    RMContext rmContext = TestUtils.getMockRMContext();
    cs.setConf(conf);
    cs.setRMContext(rmContext);
    cs.init(conf);
    //by default, the state of ALL queues should be RUNNING
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q2).getState());
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q3).getState());
    // submit an application to Q2
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    String userName = "testUser";
    cs.getQueue(Q2).submitApplication(appId, userName, Q2);
    FiCaSchedulerApp app = getMockApplication(appId, userName, Resources.createResource(4, 0));
    cs.getQueue(Q2).submitApplicationAttempt(app, userName);
    // set Q2 state to stop and do reinitialize.
    csConf.setState(Q2_PATH, QueueState.STOPPED);
    conf = new YarnConfiguration(csConf);
    cs.reinitialize(conf, rmContext);
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.DRAINING, cs.getQueue(Q2).getState());
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q3).getState());
    // set Q1 state to stop and do reinitialize.
    csConf.setState(Q1_PATH, QueueState.STOPPED);
    conf = new YarnConfiguration(csConf);
    cs.reinitialize(conf, rmContext);
    Assert.assertEquals(QueueState.DRAINING, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.DRAINING, cs.getQueue(Q2).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q3).getState());
    // Active Q3, should fail
    csConf.setState(Q3_PATH, QueueState.RUNNING);
    conf = new YarnConfiguration(csConf);
    try {
        cs.reinitialize(conf, rmContext);
        Assert.fail("Should throw an Exception.");
    } catch (Exception ex) {
    // Do Nothing
    }
    // stop the app running in q2
    cs.getQueue(Q2).finishApplicationAttempt(app, Q2);
    cs.getQueue(Q2).finishApplication(appId, userName);
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q2).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q3).getState());
}

Example 73 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestQueueState method testQueueState.

@Test(timeout = 15000)
public void testQueueState() throws IOException {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { Q1 });
    csConf.setQueues(Q1_PATH, new String[] { Q2 });
    csConf.setCapacity(Q1_PATH, 100);
    csConf.setCapacity(Q2_PATH, 100);
    conf = new YarnConfiguration(csConf);
    cs = new CapacityScheduler();
    RMContext rmContext = TestUtils.getMockRMContext();
    cs.setConf(conf);
    cs.setRMContext(rmContext);
    cs.init(conf);
    //by default, the state of both queues should be RUNNING
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q2).getState());
    // Change the state of Q1 to STOPPED, and re-initiate the CS
    csConf.setState(Q1_PATH, QueueState.STOPPED);
    conf = new YarnConfiguration(csConf);
    cs.reinitialize(conf, rmContext);
    // The state of Q1 and its child: Q2 should be STOPPED
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q2).getState());
    // Change the state of Q1 to RUNNING, and change the state of Q2 to STOPPED
    csConf.setState(Q1_PATH, QueueState.RUNNING);
    csConf.setState(Q2_PATH, QueueState.STOPPED);
    conf = new YarnConfiguration(csConf);
    // reinitialize the CS, the operation should be successful
    cs.reinitialize(conf, rmContext);
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q2).getState());
    // Change the state of Q1 to STOPPED, and change the state of Q2 to RUNNING
    csConf.setState(Q1_PATH, QueueState.STOPPED);
    csConf.setState(Q2_PATH, QueueState.RUNNING);
    conf = new YarnConfiguration(csConf);
    // reinitialize the CS, the operation should be failed.
    try {
        cs.reinitialize(conf, rmContext);
        Assert.fail("Should throw an Exception.");
    } catch (Exception ex) {
        Assert.assertTrue(ex.getCause().getMessage().contains("The parent queue:q1 state is STOPPED, " + "child queue:q2 state cannot be RUNNING."));
    }
}

Example 74 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestQueueStateManager method testQueueStateManager.

@Test
public void testQueueStateManager() throws AccessControlException, YarnException {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { Q1 });
    csConf.setQueues(Q1_PATH, new String[] { Q2, Q3 });
    csConf.setCapacity(Q1_PATH, 100);
    csConf.setCapacity(Q2_PATH, 50);
    csConf.setCapacity(Q3_PATH, 50);
    conf = new YarnConfiguration(csConf);
    cs = new CapacityScheduler();
    RMContext rmContext = TestUtils.getMockRMContext();
    cs.setConf(conf);
    cs.setRMContext(rmContext);
    cs.init(conf);
    @SuppressWarnings("rawtypes") QueueStateManager stateManager = cs.getCapacitySchedulerQueueManager().getQueueStateManager();
    //by default, the state of both queues should be RUNNING
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q2).getState());
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q3).getState());
    // Stop Q2, and verify that Q2 transmits to STOPPED STATE
    stateManager.stopQueue(Q2);
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q2).getState());
    // Stop Q1, and verify that Q1, as well as its child: Q3,
    // transmits to STOPPED STATE
    stateManager.stopQueue(Q1);
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q3).getState());
    Assert.assertTrue(stateManager.canDelete(Q1));
    Assert.assertTrue(stateManager.canDelete(Q2));
    Assert.assertTrue(stateManager.canDelete(Q3));
    // Active Q2, it will fail.
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q2).getState());
    // Now active Q1
    stateManager.activateQueue(Q1);
    // Q1 should be in RUNNING state. Its children: Q2 and Q3
    // should still be in STOPPED state.
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q2).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q3).getState());
    // Now active Q2 and Q3
    stateManager.activateQueue(Q2);
    stateManager.activateQueue(Q3);
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q2).getState());
    Assert.assertEquals(QueueState.RUNNING, cs.getQueue(Q3).getState());
    Assert.assertFalse(stateManager.canDelete(Q1));
    Assert.assertFalse(stateManager.canDelete(Q2));
    Assert.assertFalse(stateManager.canDelete(Q3));
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    String userName = "testUser";
    cs.getQueue(Q2).submitApplication(appId, userName, Q2);
    FiCaSchedulerApp app = getMockApplication(appId, userName, Resources.createResource(4, 0));
    cs.getQueue(Q2).submitApplicationAttempt(app, userName);
    stateManager.stopQueue(Q1);
    Assert.assertEquals(QueueState.DRAINING, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.DRAINING, cs.getQueue(Q2).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q3).getState());
    cs.getQueue(Q2).finishApplicationAttempt(app, Q2);
    cs.getQueue(Q2).finishApplication(appId, userName);
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q1).getState());
    Assert.assertEquals(QueueState.STOPPED, cs.getQueue(Q2).getState());
}

Example 75 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class FairSchedulerTestBase method createApplicationWithAMResource.

protected void createApplicationWithAMResource(ApplicationAttemptId attId, String queue, String user, Resource amResource) {
    RMContext rmContext = resourceManager.getRMContext();
    ApplicationId appId = attId.getApplicationId();
    RMApp rmApp = new RMAppImpl(appId, rmContext, conf, null, user, null, ApplicationSubmissionContext.newInstance(appId, null, queue, null, mock(ContainerLaunchContext.class), false, false, 0, amResource, null), scheduler, null, 0, null, null, null);
    rmContext.getRMApps().put(appId, rmApp);
    RMAppEvent event = new RMAppEvent(appId, RMAppEventType.START);
    resourceManager.getRMContext().getRMApps().get(appId).handle(event);
    event = new RMAppEvent(appId, RMAppEventType.APP_NEW_SAVED);
    resourceManager.getRMContext().getRMApps().get(appId).handle(event);
    event = new RMAppEvent(appId, RMAppEventType.APP_ACCEPTED);
    resourceManager.getRMContext().getRMApps().get(appId).handle(event);
    AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appId, queue, user);
    scheduler.handle(appAddedEvent);
    AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent(attId, false);
    scheduler.handle(attempAddedEvent);
}