Examples with Resource org.apache.hadoop.yarn.api.records.Resource used on opensource projects

Example 76 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class UsersManager method updateUserResourceUsage.

/**
   * During container allocate/release, ensure that all user specific data
   * structures are updated.
   *
   * @param userName
   *          Name of the user
   * @param resource
   *          Resource to increment/decrement
   * @param nodePartition
   *          Node label
   * @param isAllocate
   *          Indicate whether to allocate or release resource
   * @return user
   */
public User updateUserResourceUsage(String userName, Resource resource, String nodePartition, boolean isAllocate) {
    try {
        this.writeLock.lock();
        // TODO, should use getUser, use this method just to avoid UT failure
        // which is caused by wrong invoking order, will fix UT separately
        User user = getUserAndAddIfAbsent(userName);
        // New container is allocated. Invalidate user-limit.
        updateResourceUsagePerUser(user, resource, nodePartition, isAllocate);
        userLimitNeedsRecompute();
        // Update usage ratios
        Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition, scheduler.getClusterResource());
        incQueueUsageRatio(nodePartition, user.updateUsageRatio(resourceCalculator, resourceByLabel, nodePartition));
        return user;
    } finally {
        this.writeLock.unlock();
    }
}

Example 77 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class UsersManager method updateUsageRatio.

/**
   * Update new usage ratio.
   *
   * @param partition
   *          Node partition
   * @param clusterResource
   *          Cluster Resource
   */
public void updateUsageRatio(String partition, Resource clusterResource) {
    try {
        writeLock.lock();
        Resource resourceByLabel = labelManager.getResourceByLabel(partition, clusterResource);
        float consumed = 0;
        User user;
        for (Map.Entry<String, User> entry : getUsers().entrySet()) {
            user = entry.getValue();
            consumed += user.setAndUpdateUsageRatio(resourceCalculator, resourceByLabel, partition);
        }
        qUsageRatios.setUsageRatio(partition, consumed);
    } finally {
        writeLock.unlock();
    }
}

Example 78 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class UsersManager method reComputeUserLimits.

private Map<SchedulingMode, Resource> reComputeUserLimits(String userName, String nodePartition, Resource clusterResource, SchedulingMode schedulingMode, boolean activeMode) {
    // preselect stored map as per active user-limit or all user computation.
    Map<String, Map<SchedulingMode, Resource>> computedMap = null;
    computedMap = (activeMode) ? preComputedActiveUserLimit : preComputedAllUserLimit;
    Map<SchedulingMode, Resource> userLimitPerSchedulingMode = computedMap.get(nodePartition);
    if (userLimitPerSchedulingMode == null) {
        userLimitPerSchedulingMode = new ConcurrentHashMap<>();
        computedMap.put(nodePartition, userLimitPerSchedulingMode);
    }
    // compute user-limit per scheduling mode.
    Resource computedUserLimit = computeUserLimit(userName, clusterResource, nodePartition, schedulingMode, activeMode);
    // update in local storage
    userLimitPerSchedulingMode.put(schedulingMode, computedUserLimit);
    return userLimitPerSchedulingMode;
}

Example 79 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class UsersManager method computeUserLimit.

private Resource computeUserLimit(String userName, Resource clusterResource, String nodePartition, SchedulingMode schedulingMode, boolean activeUser) {
    Resource partitionResource = labelManager.getResourceByLabel(nodePartition, clusterResource);
    /*
     * What is our current capacity?
     * * It is equal to the max(required, queue-capacity) if we're running
     * below capacity. The 'max' ensures that jobs in queues with miniscule
     * capacity (< 1 slot) make progress
     * * If we're running over capacity, then its (usedResources + required)
     * (which extra resources we are allocating)
     */
    Resource queueCapacity = Resources.multiplyAndNormalizeUp(resourceCalculator, partitionResource, lQueue.getQueueCapacities().getAbsoluteCapacity(nodePartition), lQueue.getMinimumAllocation());
    /*
     * Assume we have required resource equals to minimumAllocation, this can
     * make sure user limit can continuously increase till queueMaxResource
     * reached.
     */
    Resource required = lQueue.getMinimumAllocation();
    // Allow progress for queues with miniscule capacity
    queueCapacity = Resources.max(resourceCalculator, partitionResource, queueCapacity, required);
    /*
     * We want to base the userLimit calculation on max(queueCapacity,
     * usedResources+required). However, we want usedResources to be based on
     * the combined ratios of all the users in the queue so we use consumedRatio
     * to calculate such. The calculation is dependent on how the
     * resourceCalculator calculates the ratio between two Resources. DRF
     * Example: If usedResources is greater than queueCapacity and users have
     * the following [mem,cpu] usages: User1: [10%,20%] - Dominant resource is
     * 20% User2: [30%,10%] - Dominant resource is 30% Then total consumedRatio
     * is then 20+30=50%. Yes, this value can be larger than 100% but for the
     * purposes of making sure all users are getting their fair share, it works.
     */
    Resource consumed = Resources.multiplyAndNormalizeUp(resourceCalculator, partitionResource, getUsageRatio(nodePartition), lQueue.getMinimumAllocation());
    Resource currentCapacity = Resources.lessThan(resourceCalculator, partitionResource, consumed, queueCapacity) ? queueCapacity : Resources.add(consumed, required);
    /*
     * Never allow a single user to take more than the queue's configured
     * capacity * user-limit-factor. Also, the queue's configured capacity
     * should be higher than queue-hard-limit * ulMin
     */
    int usersCount = getNumActiveUsers();
    Resource resourceUsed = totalResUsageForActiveUsers.getUsed(nodePartition);
    // For non-activeUser calculation, consider all users count.
    if (!activeUser) {
        resourceUsed = currentCapacity;
        usersCount = users.size();
    }
    /*
     * User limit resource is determined by: max{currentCapacity / #activeUsers,
     * currentCapacity * user-limit-percentage%)
     */
    Resource userLimitResource = Resources.max(resourceCalculator, partitionResource, Resources.divideAndCeil(resourceCalculator, resourceUsed, usersCount), Resources.divideAndCeil(resourceCalculator, Resources.multiplyAndRoundDown(currentCapacity, getUserLimit()), 100));
    // User limit is capped by maxUserLimit
    // - maxUserLimit = queueCapacity * user-limit-factor
    // (RESPECT_PARTITION_EXCLUSIVITY)
    // - maxUserLimit = total-partition-resource (IGNORE_PARTITION_EXCLUSIVITY)
    //
    // In IGNORE_PARTITION_EXCLUSIVITY mode, if a queue cannot access a
    // partition, its guaranteed resource on that partition is 0. And
    // user-limit-factor computation is based on queue's guaranteed capacity. So
    // we will not cap user-limit as well as used resource when doing
    // IGNORE_PARTITION_EXCLUSIVITY allocation.
    Resource maxUserLimit = Resources.none();
    if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) {
        maxUserLimit = Resources.multiplyAndRoundDown(queueCapacity, getUserLimitFactor());
    } else if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
        maxUserLimit = partitionResource;
    }
    // Cap final user limit with maxUserLimit
    userLimitResource = Resources.roundUp(resourceCalculator, Resources.min(resourceCalculator, partitionResource, userLimitResource, maxUserLimit), lQueue.getMinimumAllocation());
    if (LOG.isDebugEnabled()) {
        LOG.debug("User limit computation for " + userName + " in queue " + lQueue.getQueueName() + " userLimitPercent=" + lQueue.getUserLimit() + " userLimitFactor=" + lQueue.getUserLimitFactor() + " required: " + required + " consumed: " + consumed + " user-limit-resource: " + userLimitResource + " queueCapacity: " + queueCapacity + " qconsumed: " + lQueue.getQueueResourceUsage().getUsed() + " currentCapacity: " + currentCapacity + " activeUsers: " + usersCount + " clusterCapacity: " + clusterResource + " resourceByLabel: " + partitionResource + " usageratio: " + getUsageRatio(nodePartition) + " Partition: " + nodePartition);
    }
    getUser(userName).setUserResourceLimit(userLimitResource);
    return userLimitResource;
}

Example 80 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class AbstractContainerAllocator method getCSAssignmentFromAllocateResult.

protected CSAssignment getCSAssignmentFromAllocateResult(Resource clusterResource, ContainerAllocation result, RMContainer rmContainer, FiCaSchedulerNode node) {
    // Handle skipped
    CSAssignment.SkippedType skipped = (result.getAllocationState() == AllocationState.APP_SKIPPED) ? CSAssignment.SkippedType.OTHER : CSAssignment.SkippedType.NONE;
    CSAssignment assignment = new CSAssignment(skipped);
    assignment.setApplication(application);
    // Handle excess reservation
    assignment.setExcessReservation(result.getContainerToBeUnreserved());
    assignment.setRequestLocalityType(result.requestLocalityType);
    // If we allocated something
    if (Resources.greaterThan(rc, clusterResource, result.getResourceToBeAllocated(), Resources.none())) {
        Resource allocatedResource = result.getResourceToBeAllocated();
        RMContainer updatedContainer = result.getUpdatedContainer();
        assignment.setResource(allocatedResource);
        assignment.setType(result.getContainerNodeType());
        if (result.getAllocationState() == AllocationState.RESERVED) {
            // This is a reserved container
            LOG.info("Reserved container " + " application=" + application.getApplicationId() + " resource=" + allocatedResource + " queue=" + this.toString() + " cluster=" + clusterResource);
            assignment.getAssignmentInformation().addReservationDetails(updatedContainer, application.getCSLeafQueue().getQueuePath());
            assignment.getAssignmentInformation().incrReservations();
            Resources.addTo(assignment.getAssignmentInformation().getReserved(), allocatedResource);
            if (rmContainer != null) {
                ActivitiesLogger.APP.recordAppActivityWithAllocation(activitiesManager, node, application, updatedContainer, ActivityState.RE_RESERVED);
                ActivitiesLogger.APP.finishSkippedAppAllocationRecording(activitiesManager, application.getApplicationId(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
            } else {
                ActivitiesLogger.APP.recordAppActivityWithAllocation(activitiesManager, node, application, updatedContainer, ActivityState.RESERVED);
                ActivitiesLogger.APP.finishAllocatedAppAllocationRecording(activitiesManager, application.getApplicationId(), updatedContainer.getContainerId(), ActivityState.RESERVED, ActivityDiagnosticConstant.EMPTY);
            }
        } else if (result.getAllocationState() == AllocationState.ALLOCATED) {
            // This is a new container
            // Inform the ordering policy
            LOG.info("assignedContainer" + " application attempt=" + application.getApplicationAttemptId() + " container=" + updatedContainer.getContainerId() + " queue=" + this + " clusterResource=" + clusterResource + " type=" + assignment.getType());
            assignment.getAssignmentInformation().addAllocationDetails(updatedContainer, application.getCSLeafQueue().getQueuePath());
            assignment.getAssignmentInformation().incrAllocations();
            Resources.addTo(assignment.getAssignmentInformation().getAllocated(), allocatedResource);
            if (rmContainer != null) {
                assignment.setFulfilledReservation(true);
                assignment.setFulfilledReservedContainer(rmContainer);
            }
            ActivitiesLogger.APP.recordAppActivityWithAllocation(activitiesManager, node, application, updatedContainer, ActivityState.ALLOCATED);
            ActivitiesLogger.APP.finishAllocatedAppAllocationRecording(activitiesManager, application.getApplicationId(), updatedContainer.getContainerId(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY);
            // Update unformed resource
            application.incUnconfirmedRes(allocatedResource);
        }
        assignment.setContainersToKill(result.getToKillContainers());
    } else {
        if (result.getAllocationState() == AllocationState.QUEUE_SKIPPED) {
            assignment.setSkippedType(CSAssignment.SkippedType.QUEUE_LIMIT);
        }
    }
    return assignment;
}