Examples with Resource org.apache.hadoop.yarn.api.records.Resource used on opensource projects

Example 86 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class SchedulerApplicationAttempt method getResourceUsageReport.

public ApplicationResourceUsageReport getResourceUsageReport() {
    try {
        writeLock.lock();
        AggregateAppResourceUsage runningResourceUsage = getRunningAggregateAppResourceUsage();
        Resource usedResourceClone = Resources.clone(attemptResourceUsage.getAllUsed());
        Resource reservedResourceClone = Resources.clone(attemptResourceUsage.getReserved());
        Resource cluster = rmContext.getScheduler().getClusterResource();
        ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
        float queueUsagePerc = 0.0f;
        float clusterUsagePerc = 0.0f;
        if (!calc.isInvalidDivisor(cluster)) {
            float queueCapacityPerc = queue.getQueueInfo(false, false).getCapacity();
            if (queueCapacityPerc != 0) {
                queueUsagePerc = calc.divide(cluster, usedResourceClone, Resources.multiply(cluster, queueCapacityPerc)) * 100;
            }
            clusterUsagePerc = calc.divide(cluster, usedResourceClone, cluster) * 100;
        }
        return ApplicationResourceUsageReport.newInstance(liveContainers.size(), reservedContainers.size(), usedResourceClone, reservedResourceClone, Resources.add(usedResourceClone, reservedResourceClone), runningResourceUsage.getMemorySeconds(), runningResourceUsage.getVcoreSeconds(), queueUsagePerc, clusterUsagePerc, 0, 0);
    } finally {
        writeLock.unlock();
    }
}

Example 87 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class LeafQueue method activateApplications.

private void activateApplications() {
    try {
        writeLock.lock();
        // limit of allowed resource usage for application masters
        Map<String, Resource> userAmPartitionLimit = new HashMap<String, Resource>();
        // is initialized for the first time (when no applications are present).
        for (String nodePartition : getNodeLabelsForQueue()) {
            calculateAndGetAMResourceLimitPerPartition(nodePartition);
        }
        for (Iterator<FiCaSchedulerApp> fsApp = getPendingAppsOrderingPolicy().getAssignmentIterator(); fsApp.hasNext(); ) {
            FiCaSchedulerApp application = fsApp.next();
            ApplicationId applicationId = application.getApplicationId();
            // Get the am-node-partition associated with each application
            // and calculate max-am resource limit for this partition.
            String partitionName = application.getAppAMNodePartitionName();
            Resource amLimit = getAMResourceLimitPerPartition(partitionName);
            // Verify whether we already calculated am-limit for this label.
            if (amLimit == null) {
                amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName);
            }
            // Check am resource limit.
            Resource amIfStarted = Resources.add(application.getAMResource(partitionName), queueUsage.getAMUsed(partitionName));
            if (LOG.isDebugEnabled()) {
                LOG.debug("application " + application.getId() + " AMResource " + application.getAMResource(partitionName) + " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent + " amLimit " + amLimit + " lastClusterResource " + lastClusterResource + " amIfStarted " + amIfStarted + " AM node-partition name " + partitionName);
            }
            if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, amIfStarted, amLimit)) {
                if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, queueUsage.getAMUsed(partitionName), Resources.none()))) {
                    LOG.warn("maximum-am-resource-percent is insufficient to start a" + " single application in queue, it is likely set too low." + " skipping enforcement to allow at least one application" + " to start");
                } else {
                    application.updateAMContainerDiagnostics(AMState.INACTIVATED, CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Not activating application " + applicationId + " as  amIfStarted: " + amIfStarted + " exceeds amLimit: " + amLimit);
                    }
                    continue;
                }
            }
            // Check user am resource limit
            User user = getUser(application.getUser());
            Resource userAMLimit = userAmPartitionLimit.get(partitionName);
            // Verify whether we already calculated user-am-limit for this label.
            if (userAMLimit == null) {
                userAMLimit = getUserAMResourceLimitPerPartition(partitionName);
                userAmPartitionLimit.put(partitionName, userAMLimit);
            }
            Resource userAmIfStarted = Resources.add(application.getAMResource(partitionName), user.getConsumedAMResources(partitionName));
            if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, userAmIfStarted, userAMLimit)) {
                if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, queueUsage.getAMUsed(partitionName), Resources.none()))) {
                    LOG.warn("maximum-am-resource-percent is insufficient to start a" + " single application in queue for user, it is likely set too" + " low. skipping enforcement to allow at least one application" + " to start");
                } else {
                    application.updateAMContainerDiagnostics(AMState.INACTIVATED, CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Not activating application " + applicationId + " for user: " + user + " as userAmIfStarted: " + userAmIfStarted + " exceeds userAmLimit: " + userAMLimit);
                    }
                    continue;
                }
            }
            user.activateApplication();
            orderingPolicy.addSchedulableEntity(application);
            application.updateAMContainerDiagnostics(AMState.ACTIVATED, null);
            queueUsage.incAMUsed(partitionName, application.getAMResource(partitionName));
            user.getResourceUsage().incAMUsed(partitionName, application.getAMResource(partitionName));
            user.getResourceUsage().setAMLimit(partitionName, userAMLimit);
            metrics.incAMUsed(application.getUser(), application.getAMResource(partitionName));
            metrics.setAMResouceLimitForUser(application.getUser(), userAMLimit);
            fsApp.remove();
            LOG.info("Application " + applicationId + " from user: " + application.getUser() + " activated in queue: " + getQueueName());
        }
    } finally {
        writeLock.unlock();
    }
}

Example 88 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class LeafQueue method getHeadroom.

private Resource getHeadroom(User user, Resource currentPartitionResourceLimit, Resource clusterResource, Resource userLimitResource, String partition) {
    /** 
     * Headroom is:
     *    min(
     *        min(userLimit, queueMaxCap) - userConsumed,
     *        queueMaxCap - queueUsedResources
     *       )
     * 
     * ( which can be expressed as, 
     *  min (userLimit - userConsumed, queuMaxCap - userConsumed, 
     *    queueMaxCap - queueUsedResources)
     *  )
     *
     * given that queueUsedResources >= userConsumed, this simplifies to
     *
     * >> min (userlimit - userConsumed,   queueMaxCap - queueUsedResources) << 
     *
     * sum of queue max capacities of multiple queue's will be greater than the
     * actual capacity of a given partition, hence we need to ensure that the
     * headroom is not greater than the available resource for a given partition
     *
     * headroom = min (unused resourcelimit of a label, calculated headroom )
     */
    currentPartitionResourceLimit = partition.equals(RMNodeLabelsManager.NO_LABEL) ? currentPartitionResourceLimit : getQueueMaxResource(partition, clusterResource);
    Resource headroom = Resources.componentwiseMin(Resources.subtract(userLimitResource, user.getUsed(partition)), Resources.subtract(currentPartitionResourceLimit, queueUsage.getUsed(partition)));
    // Normalize it before return
    headroom = Resources.roundDown(resourceCalculator, headroom, minimumAllocation);
    //headroom = min (unused resourcelimit of a label, calculated headroom )
    Resource clusterPartitionResource = labelManager.getResourceByLabel(partition, clusterResource);
    Resource clusterFreePartitionResource = Resources.subtract(clusterPartitionResource, csContext.getClusterResourceUsage().getUsed(partition));
    headroom = Resources.min(resourceCalculator, clusterPartitionResource, clusterFreePartitionResource, headroom);
    return headroom;
}

Example 89 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ParentQueue method getResourceLimitsOfChild.

private ResourceLimits getResourceLimitsOfChild(CSQueue child, Resource clusterResource, Resource parentLimits, String nodePartition) {
    // Set resource-limit of a given child, child.limit =
    // min(my.limit - my.used + child.used, child.max)
    // Parent available resource = parent-limit - parent-used-resource
    Resource parentMaxAvailableResource = Resources.subtract(parentLimits, queueUsage.getUsed(nodePartition));
    // Deduct killable from used
    Resources.addTo(parentMaxAvailableResource, getTotalKillableResource(nodePartition));
    // Child's limit = parent-available-resource + child-used
    Resource childLimit = Resources.add(parentMaxAvailableResource, child.getQueueResourceUsage().getUsed(nodePartition));
    // Get child's max resource
    Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager.getResourceByLabel(nodePartition, clusterResource), child.getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition), minimumAllocation);
    // Child's limit should be capped by child configured max resource
    childLimit = Resources.min(resourceCalculator, clusterResource, childLimit, childConfiguredMaxResource);
    // Normalize before return
    childLimit = Resources.roundDown(resourceCalculator, childLimit, minimumAllocation);
    return new ResourceLimits(childLimit);
}

Example 90 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class CapacityScheduler method addNode.

private void addNode(RMNode nodeManager) {
    try {
        writeLock.lock();
        FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager, usePortForNodeName, nodeManager.getNodeLabels());
        nodeTracker.addNode(schedulerNode);
        // update this node to node label manager
        if (labelManager != null) {
            labelManager.activateNode(nodeManager.getNodeID(), schedulerNode.getTotalResource());
        }
        Resource clusterResource = getClusterResource();
        getRootQueue().updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
        LOG.info("Added node " + nodeManager.getNodeAddress() + " clusterResource: " + clusterResource);
        if (scheduleAsynchronously && getNumClusterNodes() == 1) {
            for (AsyncScheduleThread t : asyncSchedulerThreads) {
                t.beginSchedule();
            }
        }
    } finally {
        writeLock.unlock();
    }
}