Examples with User org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User used on opensource projects

Example 1 with User

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User in project hadoop by apache.

the class LeafQueue method getNumActiveApplications.

@Private
public int getNumActiveApplications(String user) {
    try {
        readLock.lock();
        User u = getUser(user);
        if (null == u) {
            return 0;
        }
        return u.getActiveApplications();
    } finally {
        readLock.unlock();
    }
}

Example 2 with User

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User in project hadoop by apache.

the class LeafQueue method activateApplications.

private void activateApplications() {
    try {
        writeLock.lock();
        // limit of allowed resource usage for application masters
        Map<String, Resource> userAmPartitionLimit = new HashMap<String, Resource>();
        // is initialized for the first time (when no applications are present).
        for (String nodePartition : getNodeLabelsForQueue()) {
            calculateAndGetAMResourceLimitPerPartition(nodePartition);
        }
        for (Iterator<FiCaSchedulerApp> fsApp = getPendingAppsOrderingPolicy().getAssignmentIterator(); fsApp.hasNext(); ) {
            FiCaSchedulerApp application = fsApp.next();
            ApplicationId applicationId = application.getApplicationId();
            // Get the am-node-partition associated with each application
            // and calculate max-am resource limit for this partition.
            String partitionName = application.getAppAMNodePartitionName();
            Resource amLimit = getAMResourceLimitPerPartition(partitionName);
            // Verify whether we already calculated am-limit for this label.
            if (amLimit == null) {
                amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName);
            }
            // Check am resource limit.
            Resource amIfStarted = Resources.add(application.getAMResource(partitionName), queueUsage.getAMUsed(partitionName));
            if (LOG.isDebugEnabled()) {
                LOG.debug("application " + application.getId() + " AMResource " + application.getAMResource(partitionName) + " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent + " amLimit " + amLimit + " lastClusterResource " + lastClusterResource + " amIfStarted " + amIfStarted + " AM node-partition name " + partitionName);
            }
            if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, amIfStarted, amLimit)) {
                if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, queueUsage.getAMUsed(partitionName), Resources.none()))) {
                    LOG.warn("maximum-am-resource-percent is insufficient to start a" + " single application in queue, it is likely set too low." + " skipping enforcement to allow at least one application" + " to start");
                } else {
                    application.updateAMContainerDiagnostics(AMState.INACTIVATED, CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Not activating application " + applicationId + " as  amIfStarted: " + amIfStarted + " exceeds amLimit: " + amLimit);
                    }
                    continue;
                }
            }
            // Check user am resource limit
            User user = getUser(application.getUser());
            Resource userAMLimit = userAmPartitionLimit.get(partitionName);
            // Verify whether we already calculated user-am-limit for this label.
            if (userAMLimit == null) {
                userAMLimit = getUserAMResourceLimitPerPartition(partitionName);
                userAmPartitionLimit.put(partitionName, userAMLimit);
            }
            Resource userAmIfStarted = Resources.add(application.getAMResource(partitionName), user.getConsumedAMResources(partitionName));
            if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, userAmIfStarted, userAMLimit)) {
                if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, queueUsage.getAMUsed(partitionName), Resources.none()))) {
                    LOG.warn("maximum-am-resource-percent is insufficient to start a" + " single application in queue for user, it is likely set too" + " low. skipping enforcement to allow at least one application" + " to start");
                } else {
                    application.updateAMContainerDiagnostics(AMState.INACTIVATED, CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Not activating application " + applicationId + " for user: " + user + " as userAmIfStarted: " + userAmIfStarted + " exceeds userAmLimit: " + userAMLimit);
                    }
                    continue;
                }
            }
            user.activateApplication();
            orderingPolicy.addSchedulableEntity(application);
            application.updateAMContainerDiagnostics(AMState.ACTIVATED, null);
            queueUsage.incAMUsed(partitionName, application.getAMResource(partitionName));
            user.getResourceUsage().incAMUsed(partitionName, application.getAMResource(partitionName));
            user.getResourceUsage().setAMLimit(partitionName, userAMLimit);
            metrics.incAMUsed(application.getUser(), application.getAMResource(partitionName));
            metrics.setAMResouceLimitForUser(application.getUser(), userAMLimit);
            fsApp.remove();
            LOG.info("Application " + applicationId + " from user: " + application.getUser() + " activated in queue: " + getQueueName());
        }
    } finally {
        writeLock.unlock();
    }
}

Example 3 with User

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User in project hadoop by apache.

the class LeafQueue method submitApplicationAttempt.

@Override
public void submitApplicationAttempt(FiCaSchedulerApp application, String userName) {
    // Careful! Locking order is important!
    try {
        writeLock.lock();
        // TODO, should use getUser, use this method just to avoid UT failure
        // which is caused by wrong invoking order, will fix UT separately
        User user = usersManager.getUserAndAddIfAbsent(userName);
        // Add the attempt to our data-structures
        addApplicationAttempt(application, user);
    } finally {
        writeLock.unlock();
    }
    // We don't want to update metrics for move app
    if (application.isPending()) {
        metrics.submitAppAttempt(userName);
    }
    getParent().submitApplicationAttempt(application, userName);
}

Example 4 with User

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User in project hadoop by apache.

the class LeafQueue method allocateResource.

void allocateResource(Resource clusterResource, SchedulerApplicationAttempt application, Resource resource, String nodePartition, RMContainer rmContainer) {
    try {
        writeLock.lock();
        super.allocateResource(clusterResource, resource, nodePartition);
        // handle ignore exclusivity container
        if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
            TreeSet<RMContainer> rmContainers = null;
            if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get(nodePartition))) {
                rmContainers = new TreeSet<>();
                ignorePartitionExclusivityRMContainers.put(nodePartition, rmContainers);
            }
            rmContainers.add(rmContainer);
        }
        // Update user metrics
        String userName = application.getUser();
        // Increment user's resource usage.
        User user = usersManager.updateUserResourceUsage(userName, resource, nodePartition, true);
        // Note this is a bit unconventional since it gets the object and modifies
        // it here, rather then using set routine
        // headroom
        Resources.subtractFrom(application.getHeadroom(), resource);
        metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
        if (LOG.isDebugEnabled()) {
            LOG.debug(getQueueName() + " user=" + userName + " used=" + queueUsage.getUsed(nodePartition) + " numContainers=" + numContainers + " headroom = " + application.getHeadroom() + " user-resources=" + user.getUsed());
        }
    } finally {
        writeLock.unlock();
    }
}

Example 5 with User

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User in project hadoop by apache.

the class LeafQueue method releaseResource.

void releaseResource(Resource clusterResource, FiCaSchedulerApp application, Resource resource, String nodePartition, RMContainer rmContainer) {
    try {
        writeLock.lock();
        super.releaseResource(clusterResource, resource, nodePartition);
        // handle ignore exclusivity container
        if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
            if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) {
                Set<RMContainer> rmContainers = ignorePartitionExclusivityRMContainers.get(nodePartition);
                rmContainers.remove(rmContainer);
                if (rmContainers.isEmpty()) {
                    ignorePartitionExclusivityRMContainers.remove(nodePartition);
                }
            }
        }
        // Update user metrics
        String userName = application.getUser();
        User user = usersManager.updateUserResourceUsage(userName, resource, nodePartition, false);
        metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
        if (LOG.isDebugEnabled()) {
            LOG.debug(getQueueName() + " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + " user=" + userName + " user-resources=" + user.getUsed());
        }
    } finally {
        writeLock.unlock();
    }
}