Examples with Resource org.apache.hadoop.yarn.api.records.Resource used on opensource projects

Example 81 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class RegularContainerAllocator method preCheckForPlacementSet.

/*
   * Pre-check if we can allocate a pending resource request
   * (given schedulerKey) to a given PlacementSet.
   * We will consider stuffs like exclusivity, pending resource, node partition,
   * headroom, etc.
   */
private ContainerAllocation preCheckForPlacementSet(Resource clusterResource, PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode, ResourceLimits resourceLimits, SchedulerRequestKey schedulerKey) {
    Priority priority = schedulerKey.getPriority();
    FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
    PendingAsk offswitchPendingAsk = application.getPendingAsk(schedulerKey, ResourceRequest.ANY);
    if (offswitchPendingAsk.getCount() <= 0) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    // Required resource
    Resource required = offswitchPendingAsk.getPerAllocationResource();
    // Do we need containers at this 'priority'?
    if (application.getOutstandingAsksCount(schedulerKey) <= 0) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.APPLICATION_PRIORITY_DO_NOT_NEED_RESOURCE);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    // avoid painful of preempt an AM container
    if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
        if (application.isWaitingForAMContainer()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip allocating AM container to app_attempt=" + application.getApplicationAttemptId() + ", don't allow to allocate AM container in non-exclusive mode");
            }
            application.updateAppSkipNodeDiagnostics("Skipping assigning to Node in Ignore Exclusivity mode. ");
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.SKIP_IN_IGNORE_EXCLUSIVITY_MODE);
            return ContainerAllocation.APP_SKIPPED;
        }
    }
    // If not match, jump to next priority.
    if (!appInfo.acceptNodePartition(schedulerKey, node.getPartition(), schedulingMode)) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NODE_PARTITION_DOES_NOT_MATCH_REQUEST);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    if (!application.getCSLeafQueue().getReservationContinueLooking()) {
        if (!shouldAllocOrReserveNewContainer(schedulerKey, required)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("doesn't need containers based on reservation algo!");
            }
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.DO_NOT_NEED_ALLOCATIONATTEMPTINFOS);
            return ContainerAllocation.PRIORITY_SKIPPED;
        }
    }
    if (!checkHeadroom(clusterResource, resourceLimits, required, ps.getPartition())) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("cannot allocate required resource=" + required + " because of headroom");
        }
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.QUEUE_SKIPPED_HEADROOM);
        return ContainerAllocation.QUEUE_SKIPPED;
    }
    // Increase missed-non-partitioned-resource-request-opportunity.
    // This is to make sure non-partitioned-resource-request will prefer
    // to be allocated to non-partitioned nodes
    int missedNonPartitionedRequestSchedulingOpportunity = 0;
    // NO_LABEL under RESPECT_EXCLUSIVITY mode
    if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL, appInfo.getSchedulingPlacementSet(schedulerKey).getPrimaryRequestedNodePartition())) {
        missedNonPartitionedRequestSchedulingOpportunity = application.addMissedNonPartitionedRequestSchedulingOpportunity(schedulerKey);
    }
    if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
        // non-partitioned partition first.
        if (missedNonPartitionedRequestSchedulingOpportunity < rmContext.getScheduler().getNumClusterNodes()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip app_attempt=" + application.getApplicationAttemptId() + " priority=" + schedulerKey.getPriority() + " because missed-non-partitioned-resource-request" + " opportunity under required:" + " Now=" + missedNonPartitionedRequestSchedulingOpportunity + " required=" + rmContext.getScheduler().getNumClusterNodes());
            }
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.NON_PARTITIONED_PARTITION_FIRST);
            return ContainerAllocation.APP_SKIPPED;
        }
    }
    return null;
}

Example 82 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ContainerUpdateContext method matchContainerToOutstandingIncreaseReq.

/**
   * Check if a new container is to be matched up against an outstanding
   * Container increase request.
   * @param node SchedulerNode.
   * @param schedulerKey SchedulerRequestKey.
   * @param rmContainer RMContainer.
   * @return ContainerId.
   */
public ContainerId matchContainerToOutstandingIncreaseReq(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer) {
    ContainerId retVal = null;
    Container container = rmContainer.getContainer();
    Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap = outstandingIncreases.get(schedulerKey);
    if (resourceMap != null) {
        Map<NodeId, Set<ContainerId>> locationMap = resourceMap.get(container.getResource());
        if (locationMap != null) {
            Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
            if (containerIds != null && !containerIds.isEmpty()) {
                retVal = containerIds.iterator().next();
            }
        }
    }
    // We also need to add these requests back.. to be reallocated.
    if (resourceMap != null && retVal == null) {
        Map<SchedulerRequestKey, Map<String, ResourceRequest>> reqsToUpdate = new HashMap<>();
        Map<String, ResourceRequest> resMap = createResourceRequests(rmContainer, node, schedulerKey, rmContainer.getContainer().getResource());
        reqsToUpdate.put(schedulerKey, resMap);
        appSchedulingInfo.addToPlacementSets(true, reqsToUpdate);
        return UNDEFINED;
    }
    return retVal;
}

Example 83 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ContainerUpdateContext method checkAndAddToOutstandingIncreases.

/**
   * Add the container to outstanding increases.
   * @param rmContainer RMContainer.
   * @param schedulerNode SchedulerNode.
   * @param updateRequest UpdateContainerRequest.
   * @return true if updated to outstanding increases was successful.
   */
public synchronized boolean checkAndAddToOutstandingIncreases(RMContainer rmContainer, SchedulerNode schedulerNode, UpdateContainerRequest updateRequest) {
    Container container = rmContainer.getContainer();
    SchedulerRequestKey schedulerKey = SchedulerRequestKey.create(updateRequest, rmContainer.getAllocatedSchedulerKey());
    Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap = outstandingIncreases.get(schedulerKey);
    if (resourceMap == null) {
        resourceMap = new HashMap<>();
        outstandingIncreases.put(schedulerKey, resourceMap);
    } else {
        // Updating Resource for and existing increase container
        if (ContainerUpdateType.INCREASE_RESOURCE == updateRequest.getContainerUpdateType()) {
            cancelPreviousRequest(schedulerNode, schedulerKey);
        } else {
            return false;
        }
    }
    Resource resToIncrease = getResourceToIncrease(updateRequest, rmContainer);
    Map<NodeId, Set<ContainerId>> locationMap = resourceMap.get(resToIncrease);
    if (locationMap == null) {
        locationMap = new HashMap<>();
        resourceMap.put(resToIncrease, locationMap);
    }
    Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
    if (containerIds == null) {
        containerIds = new HashSet<>();
        locationMap.put(container.getNodeId(), containerIds);
    }
    if (outstandingDecreases.containsKey(container.getId())) {
        return false;
    }
    containerIds.add(container.getId());
    if (!Resources.isNone(resToIncrease)) {
        Map<SchedulerRequestKey, Map<String, ResourceRequest>> updateResReqs = new HashMap<>();
        Map<String, ResourceRequest> resMap = createResourceRequests(rmContainer, schedulerNode, schedulerKey, resToIncrease);
        updateResReqs.put(schedulerKey, resMap);
        appSchedulingInfo.addToPlacementSets(false, updateResReqs);
    }
    return true;
}

Example 84 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ResourceUsage method _getAll.

private Resource _getAll(ResourceType type) {
    try {
        readLock.lock();
        Resource allOfType = Resources.createResource(0);
        for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
            //all usages types are initialized
            Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
        }
        return allOfType;
    } finally {
        readLock.unlock();
    }
}

Example 85 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ResourceUsage method getCachedDemand.

public Resource getCachedDemand(String label) {
    try {
        readLock.lock();
        Resource demand = Resources.createResource(0);
        Resources.addTo(demand, getCachedUsed(label));
        Resources.addTo(demand, getCachedPending(label));
        return demand;
    } finally {
        readLock.unlock();
    }
}