Examples with FiCaSchedulerNode org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode used on opensource projects

Example 51 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class LeafQueue method accept.

@Override
public boolean accept(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
    ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation = request.getFirstAllocatedOrReservedContainer();
    SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer = allocation.getAllocatedOrReservedContainer();
    // Do not check limits when allocation from a reserved container
    if (allocation.getAllocateFromReservedContainer() == null) {
        try {
            readLock.lock();
            FiCaSchedulerApp app = schedulerContainer.getSchedulerApplicationAttempt();
            String username = app.getUser();
            String p = schedulerContainer.getNodePartition();
            // check user-limit
            Resource userLimit = computeUserLimitAndSetHeadroom(app, cluster, p, allocation.getSchedulingMode());
            // Deduct resources that we can release
            Resource usedResource = Resources.clone(getUser(username).getUsed(p));
            Resources.subtractFrom(usedResource, request.getTotalReleasedResource());
            if (Resources.greaterThan(resourceCalculator, cluster, usedResource, userLimit)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Used resource=" + usedResource + " exceeded user-limit=" + userLimit);
                }
                return false;
            }
        } finally {
            readLock.unlock();
        }
    }
    return super.accept(cluster, request);
}

Example 52 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class LeafQueue method detachContainer.

@Override
public void detachContainer(Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer) {
    if (application != null && rmContainer != null && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
        FiCaSchedulerNode node = scheduler.getNode(rmContainer.getContainer().getNodeId());
        releaseResource(clusterResource, application, rmContainer.getContainer().getResource(), node.getPartition(), rmContainer);
        LOG.info("movedContainer" + " container=" + rmContainer.getContainer() + " resource=" + rmContainer.getContainer().getResource() + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource);
        // Inform the parent queue
        getParent().detachContainer(clusterResource, application, rmContainer);
    }
}

Example 53 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method allocateContainersToNode.

/**
   * We need to make sure when doing allocation, Node should be existed
   * And we will construct a {@link PlacementSet} before proceeding
   */
private void allocateContainersToNode(NodeId nodeId, boolean withNodeHeartbeat) {
    FiCaSchedulerNode node = getNode(nodeId);
    if (null != node) {
        int offswitchCount = 0;
        PlacementSet<FiCaSchedulerNode> ps = new SimplePlacementSet<>(node);
        CSAssignment assignment = allocateContainersToNode(ps, withNodeHeartbeat);
        // scheduling is triggered by node heartbeat
        if (null != assignment && withNodeHeartbeat) {
            if (assignment.getType() == NodeType.OFF_SWITCH) {
                offswitchCount++;
            }
            while (canAllocateMore(assignment, offswitchCount)) {
                // Try to see if it is possible to allocate multiple container for
                // the same node heartbeat
                assignment = allocateContainersToNode(ps, true);
                if (null != assignment && assignment.getType() == NodeType.OFF_SWITCH) {
                    offswitchCount++;
                }
            }
            if (offswitchCount >= offswitchPerHeartbeatLimit) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Assigned maximum number of off-switch containers: " + offswitchCount + ", assignments so far: " + assignment);
                }
            }
        }
    }
}

Example 54 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method createResourceCommitRequest.

@VisibleForTesting
public ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> createResourceCommitRequest(CSAssignment csAssignment) {
    ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocated = null;
    ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> reserved = null;
    List<SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>> released = null;
    if (Resources.greaterThan(calculator, getClusterResource(), csAssignment.getResource(), Resources.none())) {
        // Allocated something
        List<AssignmentInformation.AssignmentDetails> allocations = csAssignment.getAssignmentInformation().getAllocationDetails();
        if (!allocations.isEmpty()) {
            RMContainer rmContainer = allocations.get(0).rmContainer;
            allocated = new ContainerAllocationProposal<>(getSchedulerContainer(rmContainer, true), getSchedulerContainersToRelease(csAssignment), getSchedulerContainer(csAssignment.getFulfilledReservedContainer(), false), csAssignment.getType(), csAssignment.getRequestLocalityType(), csAssignment.getSchedulingMode() != null ? csAssignment.getSchedulingMode() : SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, csAssignment.getResource());
        }
        // Reserved something
        List<AssignmentInformation.AssignmentDetails> reservation = csAssignment.getAssignmentInformation().getReservationDetails();
        if (!reservation.isEmpty()) {
            RMContainer rmContainer = reservation.get(0).rmContainer;
            reserved = new ContainerAllocationProposal<>(getSchedulerContainer(rmContainer, false), getSchedulerContainersToRelease(csAssignment), getSchedulerContainer(csAssignment.getFulfilledReservedContainer(), false), csAssignment.getType(), csAssignment.getRequestLocalityType(), csAssignment.getSchedulingMode() != null ? csAssignment.getSchedulingMode() : SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, csAssignment.getResource());
        }
    }
    // kill all to-release containers in the request.
    if (null == allocated && null == reserved) {
        released = getSchedulerContainersToRelease(csAssignment);
    }
    if (null != allocated || null != reserved || (null != released && !released.isEmpty())) {
        List<ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>> allocationsList = null;
        if (allocated != null) {
            allocationsList = new ArrayList<>();
            allocationsList.add(allocated);
        }
        List<ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>> reservationsList = null;
        if (reserved != null) {
            reservationsList = new ArrayList<>();
            reservationsList.add(reserved);
        }
        return new ResourceCommitRequest<>(allocationsList, reservationsList, released);
    }
    return null;
}

Example 55 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method markContainerForKillable.

public void markContainerForKillable(RMContainer killableContainer) {
    try {
        writeLock.lock();
        if (LOG.isDebugEnabled()) {
            LOG.debug(SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE + ": container" + killableContainer.toString());
        }
        if (!isLazyPreemptionEnabled) {
            super.completedContainer(killableContainer, SchedulerUtils.createPreemptedContainerStatus(killableContainer.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL);
        } else {
            FiCaSchedulerNode node = (FiCaSchedulerNode) getSchedulerNode(killableContainer.getAllocatedNode());
            FiCaSchedulerApp application = getCurrentAttemptForContainer(killableContainer.getContainerId());
            node.markContainerToKillable(killableContainer.getContainerId());
            // Get the application for the finished container
            if (null != application) {
                String leafQueueName = application.getCSLeafQueue().getQueueName();
                getPreemptionManager().addKillableContainer(new KillableContainer(killableContainer, node.getPartition(), leafQueueName));
            }
        }
    } finally {
        writeLock.unlock();
    }
}