Examples with FiCaSchedulerNode org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode used on opensource projects

Example 11 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method tryCommit.

@Override
public void tryCommit(Resource cluster, ResourceCommitRequest r) {
    ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request = (ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>) r;
    ApplicationAttemptId attemptId = null;
    // We need to update unconfirmed allocated resource of application when
    // any container allocated.
    boolean updateUnconfirmedAllocatedResource = request.getContainersToAllocate() != null && !request.getContainersToAllocate().isEmpty();
    // find the application to accept and apply the ResourceCommitRequest
    if (request.anythingAllocatedOrReserved()) {
        ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> c = request.getFirstAllocatedOrReservedContainer();
        attemptId = c.getAllocatedOrReservedContainer().getSchedulerApplicationAttempt().getApplicationAttemptId();
    } else {
        if (!request.getContainersToRelease().isEmpty()) {
            attemptId = request.getContainersToRelease().get(0).getSchedulerApplicationAttempt().getApplicationAttemptId();
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Try to commit allocation proposal=" + request);
    }
    if (attemptId != null) {
        FiCaSchedulerApp app = getApplicationAttempt(attemptId);
        if (app != null) {
            if (app.accept(cluster, request)) {
                app.apply(cluster, request);
                LOG.info("Allocation proposal accepted");
            } else {
                LOG.info("Failed to accept allocation proposal");
            }
            // Update unconfirmed allocated resource.
            if (updateUnconfirmedAllocatedResource) {
                app.decUnconfirmedRes(request.getTotalAllocatedResource());
            }
        }
    }
}

Example 12 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method completedContainerInternal.

@Override
protected void completedContainerInternal(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) {
    Container container = rmContainer.getContainer();
    ContainerId containerId = container.getId();
    // Get the application for the finished container
    FiCaSchedulerApp application = getCurrentAttemptForContainer(container.getId());
    ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
    if (application == null) {
        LOG.info("Container " + container + " of" + " finished application " + appId + " completed with event " + event);
        return;
    }
    // Get the node on which the container was allocated
    FiCaSchedulerNode node = getNode(container.getNodeId());
    if (null == node) {
        LOG.info("Container " + container + " of" + " removed node " + container.getNodeId() + " completed with event " + event);
        return;
    }
    // Inform the queue
    LeafQueue queue = (LeafQueue) application.getQueue();
    queue.completedContainer(getClusterResource(), application, node, rmContainer, containerStatus, event, null, true);
}

Example 13 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method schedule.

/**
   * Schedule on all nodes by starting at a random point.
   * @param cs
   */
static void schedule(CapacityScheduler cs) {
    // First randomize the start point
    int current = 0;
    Collection<FiCaSchedulerNode> nodes = cs.nodeTracker.getAllNodes();
    int start = random.nextInt(nodes.size());
    for (FiCaSchedulerNode node : nodes) {
        if (current++ >= start) {
            cs.allocateContainersToNode(node.getNodeID(), false);
        }
    }
    // Now, just get everyone to be safe
    for (FiCaSchedulerNode node : nodes) {
        cs.allocateContainersToNode(node.getNodeID(), false);
    }
    try {
        Thread.sleep(cs.getAsyncScheduleInterval());
    } catch (InterruptedException e) {
    }
}

Example 14 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class RegularContainerAllocator method assignContainers.

@Override
public CSAssignment assignContainers(Resource clusterResource, PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode, ResourceLimits resourceLimits, RMContainer reservedContainer) {
    FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
    if (reservedContainer == null) {
        // Check if application needs more resource, skip if it doesn't need more.
        if (!application.hasPendingResourceRequest(rc, ps.getPartition(), clusterResource, schedulingMode)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip app_attempt=" + application.getApplicationAttemptId() + ", because it doesn't need more resource, schedulingMode=" + schedulingMode.name() + " node-label=" + ps.getPartition());
            }
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.APPLICATION_DO_NOT_NEED_RESOURCE);
            return CSAssignment.SKIP_ASSIGNMENT;
        }
        // Schedule in priority order
        for (SchedulerRequestKey schedulerKey : application.getSchedulerKeys()) {
            ContainerAllocation result = allocate(clusterResource, ps, schedulingMode, resourceLimits, schedulerKey, null);
            AllocationState allocationState = result.getAllocationState();
            if (allocationState == AllocationState.PRIORITY_SKIPPED) {
                continue;
            }
            return getCSAssignmentFromAllocateResult(clusterResource, result, null, node);
        }
        // We will reach here if we skipped all priorities of the app, so we will
        // skip the app.
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.SKIPPED_ALL_PRIORITIES);
        return CSAssignment.SKIP_ASSIGNMENT;
    } else {
        ContainerAllocation result = allocate(clusterResource, ps, schedulingMode, resourceLimits, reservedContainer.getReservedSchedulerKey(), reservedContainer);
        return getCSAssignmentFromAllocateResult(clusterResource, result, reservedContainer, node);
    }
}

Example 15 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class LeafQueue method assignContainers.

@Override
public CSAssignment assignContainers(Resource clusterResource, PlacementSet<FiCaSchedulerNode> ps, ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) {
    updateCurrentResourceLimits(currentResourceLimits, clusterResource);
    FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
    if (LOG.isDebugEnabled()) {
        LOG.debug("assignContainers: partition=" + ps.getPartition() + " #applications=" + orderingPolicy.getNumSchedulableEntities());
    }
    setPreemptionAllowed(currentResourceLimits, ps.getPartition());
    // Check for reserved resources, try to allocate reserved container first.
    CSAssignment assignment = allocateFromReservedContainer(clusterResource, ps, currentResourceLimits, schedulingMode);
    if (null != assignment) {
        return assignment;
    }
    // if our queue cannot access this node, just return
    if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY && !accessibleToPartition(ps.getPartition())) {
        ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, getParent().getQueueName(), getQueueName(), ActivityState.REJECTED, ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + ps.getPartition());
        return CSAssignment.NULL_ASSIGNMENT;
    }
    // queue doesn't need more resources.
    if (!hasPendingResourceRequest(ps.getPartition(), clusterResource, schedulingMode)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Skip this queue=" + getQueuePath() + ", because it doesn't need more resource, schedulingMode=" + schedulingMode.name() + " node-partition=" + ps.getPartition());
        }
        ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE);
        return CSAssignment.NULL_ASSIGNMENT;
    }
    for (Iterator<FiCaSchedulerApp> assignmentIterator = orderingPolicy.getAssignmentIterator(); assignmentIterator.hasNext(); ) {
        FiCaSchedulerApp application = assignmentIterator.next();
        ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager, node.getNodeID(), SystemClock.getInstance().getTime(), application);
        // Check queue max-capacity limit
        if (!super.canAssignToThisQueue(clusterResource, ps.getPartition(), currentResourceLimits, application.getCurrentReservation(), schedulingMode)) {
            ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.QUEUE_MAX_CAPACITY_LIMIT);
            ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
            return CSAssignment.NULL_ASSIGNMENT;
        }
        Resource userLimit = computeUserLimitAndSetHeadroom(application, clusterResource, ps.getPartition(), schedulingMode);
        // Check user limit
        if (!canAssignToUser(clusterResource, application.getUser(), userLimit, application, ps.getPartition(), currentResourceLimits)) {
            application.updateAMContainerDiagnostics(AMState.ACTIVATED, "User capacity has reached its maximum limit.");
            ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.USER_CAPACITY_MAXIMUM_LIMIT);
            continue;
        }
        // Try to schedule
        assignment = application.assignContainers(clusterResource, ps, currentResourceLimits, schedulingMode, null);
        if (LOG.isDebugEnabled()) {
            LOG.debug("post-assignContainers for application " + application.getApplicationId());
            application.showRequests();
        }
        // Did we schedule or reserve a container?
        Resource assigned = assignment.getResource();
        if (Resources.greaterThan(resourceCalculator, clusterResource, assigned, Resources.none())) {
            ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, getParent().getQueueName(), getQueueName(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY);
            return assignment;
        } else if (assignment.getSkippedType() == CSAssignment.SkippedType.OTHER) {
            ActivitiesLogger.APP.finishSkippedAppAllocationRecording(activitiesManager, application.getApplicationId(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
            application.updateNodeInfoForAMDiagnostics(node);
        } else if (assignment.getSkippedType() == CSAssignment.SkippedType.QUEUE_LIMIT) {
            return assignment;
        } else {
            // If we don't allocate anything, and it is not skipped by application,
            // we will return to respect FIFO of applications
            ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, ActivityDiagnosticConstant.RESPECT_FIFO);
            ActivitiesLogger.APP.finishSkippedAppAllocationRecording(activitiesManager, application.getApplicationId(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
            return CSAssignment.NULL_ASSIGNMENT;
        }
    }
    ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
    return CSAssignment.NULL_ASSIGNMENT;
}