Examples with RMContext org.apache.hadoop.yarn.server.resourcemanager.RMContext used on opensource projects

Example 66 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestRMAppTransitions method setUp.

@Before
public void setUp() throws Exception {
    conf = new YarnConfiguration();
    AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
    if (isSecurityEnabled) {
        authMethod = AuthenticationMethod.KERBEROS;
    }
    SecurityUtil.setAuthenticationMethod(authMethod, conf);
    UserGroupInformation.setConfiguration(conf);
    rmDispatcher = new DrainDispatcher();
    ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class);
    AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
    AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
    store = mock(RMStateStore.class);
    writer = mock(RMApplicationHistoryWriter.class);
    DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
    RMContext realRMContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, renewer, new AMRMTokenSecretManager(conf, this.rmContext), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM());
    ((RMContextImpl) realRMContext).setStateStore(store);
    publisher = mock(SystemMetricsPublisher.class);
    realRMContext.setSystemMetricsPublisher(publisher);
    realRMContext.setRMApplicationHistoryWriter(writer);
    this.rmContext = spy(realRMContext);
    ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
    doReturn(null).when(resourceScheduler).getAppResourceUsageReport((ApplicationAttemptId) Matchers.any());
    doReturn(resourceScheduler).when(rmContext).getScheduler();
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher(this.rmContext));
    rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher(rmContext));
    rmDispatcher.register(RMAppManagerEventType.class, new TestApplicationManagerEventDispatcher());
    schedulerDispatcher = new TestSchedulerEventDispatcher();
    rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher);
    rmDispatcher.init(conf);
    rmDispatcher.start();
}

Example 67 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestSchedulerApplicationAttempt method testMove.

@Test
public void testMove() {
    final String user = "user1";
    Queue parentQueue = createQueue("parent", null);
    Queue oldQueue = createQueue("old", parentQueue);
    Queue newQueue = createQueue("new", parentQueue);
    QueueMetrics parentMetrics = parentQueue.getMetrics();
    QueueMetrics oldMetrics = oldQueue.getMetrics();
    QueueMetrics newMetrics = newQueue.getMetrics();
    ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getEpoch()).thenReturn(3L);
    SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, oldQueue, oldQueue.getAbstractUsersManager(), rmContext);
    oldMetrics.submitApp(user);
    // confirm that containerId is calculated based on epoch.
    assertEquals(0x30000000001L, app.getNewContainerId());
    // Resource request
    Resource requestedResource = Resource.newInstance(1536, 2);
    Priority requestedPriority = Priority.newInstance(2);
    ResourceRequest request = ResourceRequest.newInstance(requestedPriority, ResourceRequest.ANY, requestedResource, 3);
    app.updateResourceRequests(Arrays.asList(request));
    // Allocated container
    RMContainer container1 = createRMContainer(appAttId, 1, requestedResource);
    app.liveContainers.put(container1.getContainerId(), container1);
    SchedulerNode node = createNode();
    app.appSchedulingInfo.allocate(NodeType.OFF_SWITCH, node, toSchedulerKey(requestedPriority), container1.getContainer());
    // Reserved container
    Priority prio1 = Priority.newInstance(1);
    Resource reservedResource = Resource.newInstance(2048, 3);
    RMContainer container2 = createReservedRMContainer(appAttId, 1, reservedResource, node.getNodeID(), prio1);
    Map<NodeId, RMContainer> reservations = new HashMap<NodeId, RMContainer>();
    reservations.put(node.getNodeID(), container2);
    app.reservedContainers.put(toSchedulerKey(prio1), reservations);
    oldMetrics.reserveResource(user, reservedResource);
    checkQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
    checkQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
    checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
    app.move(newQueue);
    checkQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
    checkQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
    checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
}

Example 68 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestSchedulerApplicationAttempt method testAppPercentages.

@Test
public void testAppPercentages() throws Exception {
    FifoScheduler scheduler = mock(FifoScheduler.class);
    when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(10 * 1024, 10));
    when(scheduler.getResourceCalculator()).thenReturn(new DefaultResourceCalculator());
    ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getEpoch()).thenReturn(3L);
    when(rmContext.getScheduler()).thenReturn(scheduler);
    final String user = "user1";
    Queue queue = createQueue("test", null);
    SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    // Resource request
    Resource requestedResource = Resource.newInstance(1536, 2);
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(15.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
    queue = createQueue("test2", null, 0.5f);
    app = new SchedulerApplicationAttempt(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(30.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
    app.attemptResourceUsage.incUsed(requestedResource);
    app.attemptResourceUsage.incUsed(requestedResource);
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(120.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(60.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
    queue = createQueue("test3", null, 0.0f);
    app = new SchedulerApplicationAttempt(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    // Resource request
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(0.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.01f);
    assertEquals(15.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
}

Example 69 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestSchedulerApplicationAttempt method testAppPercentagesOnswitch.

@Test
public void testAppPercentagesOnswitch() throws Exception {
    FifoScheduler scheduler = mock(FifoScheduler.class);
    when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(0, 0));
    when(scheduler.getResourceCalculator()).thenReturn(new DefaultResourceCalculator());
    ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getEpoch()).thenReturn(3L);
    when(rmContext.getScheduler()).thenReturn(scheduler);
    final String user = "user1";
    Queue queue = createQueue("test", null);
    SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext);
    // Resource request
    Resource requestedResource = Resource.newInstance(1536, 2);
    app.attemptResourceUsage.incUsed(requestedResource);
    assertEquals(0.0f, app.getResourceUsageReport().getQueueUsagePercentage(), 0.0f);
    assertEquals(0.0f, app.getResourceUsageReport().getClusterUsagePercentage(), 0.0f);
}

Example 70 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestSchedulerApplicationAttempt method testSchedulingOpportunityOverflow.

@Test
public void testSchedulingOpportunityOverflow() throws Exception {
    ApplicationAttemptId attemptId = createAppAttemptId(0, 0);
    Queue queue = createQueue("test", null);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getEpoch()).thenReturn(3L);
    SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(attemptId, "user", queue, queue.getAbstractUsersManager(), rmContext);
    Priority priority = Priority.newInstance(1);
    SchedulerRequestKey schedulerKey = toSchedulerKey(priority);
    assertEquals(0, app.getSchedulingOpportunities(schedulerKey));
    app.addSchedulingOpportunity(schedulerKey);
    assertEquals(1, app.getSchedulingOpportunities(schedulerKey));
    // verify the count is capped at MAX_VALUE and does not overflow
    app.setSchedulingOpportunities(schedulerKey, Integer.MAX_VALUE - 1);
    assertEquals(Integer.MAX_VALUE - 1, app.getSchedulingOpportunities(schedulerKey));
    app.addSchedulingOpportunity(schedulerKey);
    assertEquals(Integer.MAX_VALUE, app.getSchedulingOpportunities(schedulerKey));
    app.addSchedulingOpportunity(schedulerKey);
    assertEquals(Integer.MAX_VALUE, app.getSchedulingOpportunities(schedulerKey));
}