Examples with ActiveUsersManager org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager used on opensource projects

Example 1 with ActiveUsersManager

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager in project hadoop by apache.

the class FifoScheduler method initScheduler.

private synchronized void initScheduler(Configuration conf) {
    validateConf(conf);
    //Use ConcurrentSkipListMap because applications need to be ordered
    this.applications = new ConcurrentSkipListMap<>();
    this.minimumAllocation = Resources.createResource(conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
    initMaximumResourceCapability(Resources.createResource(conf.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB), conf.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
    this.usePortForNodeName = conf.getBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
    this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, conf);
    this.activeUsersManager = new ActiveUsersManager(metrics);
}

Example 2 with ActiveUsersManager

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager in project hadoop by apache.

the class TestLeafQueue method createQueue.

private AbstractCSQueue createQueue(String name, Queue parent, float capacity, float absCap) {
    CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
    QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null, null, QueueState.RUNNING, null, "", null, false);
    ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
    AbstractCSQueue queue = mock(AbstractCSQueue.class);
    when(queue.getMetrics()).thenReturn(metrics);
    when(queue.getAbstractUsersManager()).thenReturn(activeUsersManager);
    when(queue.getQueueInfo(false, false)).thenReturn(queueInfo);
    QueueCapacities qCaps = mock(QueueCapacities.class);
    when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap);
    when(queue.getQueueCapacities()).thenReturn(qCaps);
    return queue;
}

Example 3 with ActiveUsersManager

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager in project hadoop by apache.

the class TestApplicationLimits method testAMResourceLimit.

@Test
public void testAMResourceLimit() throws Exception {
    final String user_0 = "user_0";
    final String user_1 = "user_1";
    // This uses the default 10% of cluster value for the max am resources
    // which are allowed, at 80GB = 8GB for AM's at the queue level.  The user
    // am limit is 4G initially (based on the queue absolute capacity)
    // when there is only 1 user, and drops to 2G (the userlimit) when there
    // is a second user
    Resource clusterResource = Resource.newInstance(80 * GB, 40);
    queue.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
    ActiveUsersManager activeUsersManager = mock(ActiveUsersManager.class);
    when(queue.getAbstractUsersManager()).thenReturn(activeUsersManager);
    assertEquals(Resource.newInstance(8 * GB, 1), queue.calculateAndGetAMResourceLimit());
    assertEquals(Resource.newInstance(4 * GB, 1), queue.getUserAMResourceLimit());
    // Two apps for user_0, both start
    int APPLICATION_ID = 0;
    FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0, Resource.newInstance(2 * GB, 1));
    queue.submitApplicationAttempt(app_0, user_0);
    assertEquals(1, queue.getNumActiveApplications());
    assertEquals(0, queue.getNumPendingApplications());
    assertEquals(1, queue.getNumActiveApplications(user_0));
    assertEquals(0, queue.getNumPendingApplications(user_0));
    when(activeUsersManager.getNumActiveUsers()).thenReturn(1);
    FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0, Resource.newInstance(2 * GB, 1));
    queue.submitApplicationAttempt(app_1, user_0);
    assertEquals(2, queue.getNumActiveApplications());
    assertEquals(0, queue.getNumPendingApplications());
    assertEquals(2, queue.getNumActiveApplications(user_0));
    assertEquals(0, queue.getNumPendingApplications(user_0));
    // AMLimits unchanged
    assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
    assertEquals(Resource.newInstance(4 * GB, 1), queue.getUserAMResourceLimit());
    // One app for user_1, starts
    FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_1, Resource.newInstance(2 * GB, 1));
    queue.submitApplicationAttempt(app_2, user_1);
    assertEquals(3, queue.getNumActiveApplications());
    assertEquals(0, queue.getNumPendingApplications());
    assertEquals(1, queue.getNumActiveApplications(user_1));
    assertEquals(0, queue.getNumPendingApplications(user_1));
    when(activeUsersManager.getNumActiveUsers()).thenReturn(2);
    // Now userAMResourceLimit drops to the queue configured 50% as there is
    // another user active
    assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
    assertEquals(Resource.newInstance(2 * GB, 1), queue.getUserAMResourceLimit());
    // Second user_1 app cannot start
    FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_1, Resource.newInstance(2 * GB, 1));
    queue.submitApplicationAttempt(app_3, user_1);
    assertEquals(3, queue.getNumActiveApplications());
    assertEquals(1, queue.getNumPendingApplications());
    assertEquals(1, queue.getNumActiveApplications(user_1));
    assertEquals(1, queue.getNumPendingApplications(user_1));
    // Now finish app so another should be activated
    queue.finishApplicationAttempt(app_2, A);
    assertEquals(3, queue.getNumActiveApplications());
    assertEquals(0, queue.getNumPendingApplications());
    assertEquals(1, queue.getNumActiveApplications(user_1));
    assertEquals(0, queue.getNumPendingApplications(user_1));
}

Example 4 with ActiveUsersManager

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager in project hadoop by apache.

the class FairScheduler method addApplicationAttempt.

/**
   * Add a new application attempt to the scheduler.
   */
protected void addApplicationAttempt(ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering) {
    try {
        writeLock.lock();
        SchedulerApplication<FSAppAttempt> application = applications.get(applicationAttemptId.getApplicationId());
        String user = application.getUser();
        FSLeafQueue queue = (FSLeafQueue) application.getQueue();
        FSAppAttempt attempt = new FSAppAttempt(this, applicationAttemptId, user, queue, new ActiveUsersManager(getRootQueueMetrics()), rmContext);
        if (transferStateFromPreviousAttempt) {
            attempt.transferStateFromPreviousAttempt(application.getCurrentAppAttempt());
        }
        application.setCurrentAppAttempt(attempt);
        boolean runnable = maxRunningEnforcer.canAppBeRunnable(queue, attempt);
        queue.addApp(attempt, runnable);
        if (runnable) {
            maxRunningEnforcer.trackRunnableApp(attempt);
        } else {
            maxRunningEnforcer.trackNonRunnableApp(attempt);
        }
        queue.getMetrics().submitAppAttempt(user);
        LOG.info("Added Application Attempt " + applicationAttemptId + " to scheduler from user: " + user);
        if (isAttemptRecovering) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(applicationAttemptId + " is recovering. Skipping notifying ATTEMPT_ADDED");
            }
        } else {
            rmContext.getDispatcher().getEventHandler().handle(new RMAppAttemptEvent(applicationAttemptId, RMAppAttemptEventType.ATTEMPT_ADDED));
        }
    } finally {
        writeLock.unlock();
    }
}