Examples with PendingAsk org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk used on opensource projects

Example 1 with PendingAsk

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk in project hadoop by apache.

the class RegularContainerAllocator method preCheckForPlacementSet.

/*
   * Pre-check if we can allocate a pending resource request
   * (given schedulerKey) to a given PlacementSet.
   * We will consider stuffs like exclusivity, pending resource, node partition,
   * headroom, etc.
   */
private ContainerAllocation preCheckForPlacementSet(Resource clusterResource, PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode, ResourceLimits resourceLimits, SchedulerRequestKey schedulerKey) {
    Priority priority = schedulerKey.getPriority();
    FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
    PendingAsk offswitchPendingAsk = application.getPendingAsk(schedulerKey, ResourceRequest.ANY);
    if (offswitchPendingAsk.getCount() <= 0) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    // Required resource
    Resource required = offswitchPendingAsk.getPerAllocationResource();
    // Do we need containers at this 'priority'?
    if (application.getOutstandingAsksCount(schedulerKey) <= 0) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.APPLICATION_PRIORITY_DO_NOT_NEED_RESOURCE);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    // avoid painful of preempt an AM container
    if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
        if (application.isWaitingForAMContainer()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip allocating AM container to app_attempt=" + application.getApplicationAttemptId() + ", don't allow to allocate AM container in non-exclusive mode");
            }
            application.updateAppSkipNodeDiagnostics("Skipping assigning to Node in Ignore Exclusivity mode. ");
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.SKIP_IN_IGNORE_EXCLUSIVITY_MODE);
            return ContainerAllocation.APP_SKIPPED;
        }
    }
    // If not match, jump to next priority.
    if (!appInfo.acceptNodePartition(schedulerKey, node.getPartition(), schedulingMode)) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NODE_PARTITION_DOES_NOT_MATCH_REQUEST);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    if (!application.getCSLeafQueue().getReservationContinueLooking()) {
        if (!shouldAllocOrReserveNewContainer(schedulerKey, required)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("doesn't need containers based on reservation algo!");
            }
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.DO_NOT_NEED_ALLOCATIONATTEMPTINFOS);
            return ContainerAllocation.PRIORITY_SKIPPED;
        }
    }
    if (!checkHeadroom(clusterResource, resourceLimits, required, ps.getPartition())) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("cannot allocate required resource=" + required + " because of headroom");
        }
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.QUEUE_SKIPPED_HEADROOM);
        return ContainerAllocation.QUEUE_SKIPPED;
    }
    // Increase missed-non-partitioned-resource-request-opportunity.
    // This is to make sure non-partitioned-resource-request will prefer
    // to be allocated to non-partitioned nodes
    int missedNonPartitionedRequestSchedulingOpportunity = 0;
    // NO_LABEL under RESPECT_EXCLUSIVITY mode
    if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL, appInfo.getSchedulingPlacementSet(schedulerKey).getPrimaryRequestedNodePartition())) {
        missedNonPartitionedRequestSchedulingOpportunity = application.addMissedNonPartitionedRequestSchedulingOpportunity(schedulerKey);
    }
    if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
        // non-partitioned partition first.
        if (missedNonPartitionedRequestSchedulingOpportunity < rmContext.getScheduler().getNumClusterNodes()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip app_attempt=" + application.getApplicationAttemptId() + " priority=" + schedulerKey.getPriority() + " because missed-non-partitioned-resource-request" + " opportunity under required:" + " Now=" + missedNonPartitionedRequestSchedulingOpportunity + " required=" + rmContext.getScheduler().getNumClusterNodes());
            }
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.NON_PARTITIONED_PARTITION_FIRST);
            return ContainerAllocation.APP_SKIPPED;
        }
    }
    return null;
}

Example 2 with PendingAsk

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk in project hadoop by apache.

the class RegularContainerAllocator method assignContainersOnNode.

private ContainerAllocation assignContainersOnNode(Resource clusterResource, FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer reservedContainer, SchedulingMode schedulingMode, ResourceLimits currentResoureLimits) {
    Priority priority = schedulerKey.getPriority();
    ContainerAllocation allocation;
    NodeType requestLocalityType = null;
    // Data-local
    PendingAsk nodeLocalAsk = application.getPendingAsk(schedulerKey, node.getNodeName());
    if (nodeLocalAsk.getCount() > 0) {
        requestLocalityType = NodeType.NODE_LOCAL;
        allocation = assignNodeLocalContainers(clusterResource, nodeLocalAsk, node, schedulerKey, reservedContainer, schedulingMode, currentResoureLimits);
        if (Resources.greaterThan(rc, clusterResource, allocation.getResourceToBeAllocated(), Resources.none())) {
            allocation.requestLocalityType = requestLocalityType;
            return allocation;
        }
    }
    // Rack-local
    PendingAsk rackLocalAsk = application.getPendingAsk(schedulerKey, node.getRackName());
    if (rackLocalAsk.getCount() > 0) {
        if (!appInfo.canDelayTo(schedulerKey, node.getRackName())) {
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.SKIP_PRIORITY_BECAUSE_OF_RELAX_LOCALITY);
            return ContainerAllocation.PRIORITY_SKIPPED;
        }
        requestLocalityType = requestLocalityType == null ? NodeType.RACK_LOCAL : requestLocalityType;
        allocation = assignRackLocalContainers(clusterResource, rackLocalAsk, node, schedulerKey, reservedContainer, schedulingMode, currentResoureLimits);
        if (Resources.greaterThan(rc, clusterResource, allocation.getResourceToBeAllocated(), Resources.none())) {
            allocation.requestLocalityType = requestLocalityType;
            return allocation;
        }
    }
    // Off-switch
    PendingAsk offSwitchAsk = application.getPendingAsk(schedulerKey, ResourceRequest.ANY);
    if (offSwitchAsk.getCount() > 0) {
        if (!appInfo.canDelayTo(schedulerKey, ResourceRequest.ANY)) {
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.SKIP_PRIORITY_BECAUSE_OF_RELAX_LOCALITY);
            return ContainerAllocation.PRIORITY_SKIPPED;
        }
        requestLocalityType = requestLocalityType == null ? NodeType.OFF_SWITCH : requestLocalityType;
        allocation = assignOffSwitchContainers(clusterResource, offSwitchAsk, node, schedulerKey, reservedContainer, schedulingMode, currentResoureLimits);
        // off-switch request now, we will skip this app w.r.t priorities 
        if (allocation.state == AllocationState.LOCALITY_SKIPPED) {
            allocation.state = AllocationState.APP_SKIPPED;
        }
        allocation.requestLocalityType = requestLocalityType;
        return allocation;
    }
    ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.PRIORITY_SKIPPED);
    return ContainerAllocation.PRIORITY_SKIPPED;
}

Example 3 with PendingAsk

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk in project hadoop by apache.

the class FSAppAttempt method updateDemand.

@Override
public void updateDemand() {
    demand = Resources.createResource(0);
    // Demand is current consumption plus outstanding requests
    Resources.addTo(demand, getCurrentConsumption());
    // Add up outstanding resource requests
    try {
        writeLock.lock();
        for (SchedulerRequestKey k : getSchedulerKeys()) {
            PendingAsk pendingAsk = getPendingAsk(k, ResourceRequest.ANY);
            if (pendingAsk.getCount() > 0) {
                Resources.multiplyAndAddTo(demand, pendingAsk.getPerAllocationResource(), pendingAsk.getCount());
            }
        }
    } finally {
        writeLock.unlock();
    }
}

Example 4 with PendingAsk

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk in project hadoop by apache.

the class FSAppAttempt method assignContainer.

@Override
public Resource assignContainer(FSSchedulerNode node) {
    if (isOverAMShareLimit()) {
        PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk();
        updateAMDiagnosticMsg(amAsk.getPerAllocationResource(), " exceeds maximum AM resource allowed).");
        if (LOG.isDebugEnabled()) {
            LOG.debug("AM resource request: " + amAsk.getPerAllocationResource() + " exceeds maximum AM resource allowed, " + getQueue().dumpState());
        }
        return Resources.none();
    }
    return assignContainer(node, false);
}

Example 5 with PendingAsk

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk in project hadoop by apache.

the class FifoScheduler method assignRackLocalContainers.

private int assignRackLocalContainers(FiCaSchedulerNode node, FifoAppAttempt application, SchedulerRequestKey schedulerKey) {
    int assignedContainers = 0;
    PendingAsk rackAsk = application.getPendingAsk(schedulerKey, node.getRMNode().getRackName());
    if (rackAsk.getCount() > 0) {
        // Don't allocate on this rack if the application doens't need containers
        if (application.getOutstandingAsksCount(schedulerKey, ResourceRequest.ANY) <= 0) {
            return 0;
        }
        int assignableContainers = Math.min(getMaxAllocatableContainers(application, schedulerKey, node, NodeType.RACK_LOCAL), rackAsk.getCount());
        assignedContainers = assignContainer(node, application, schedulerKey, assignableContainers, rackAsk.getPerAllocationResource(), NodeType.RACK_LOCAL);
    }
    return assignedContainers;
}