Examples with FiCaSchedulerNode org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode used on opensource projects

Example 61 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method buildEnv.

public void buildEnv(String labelsConfig, String nodesConfig, String queuesConfig, String appsConfig, boolean useDominantResourceCalculator) throws IOException {
    if (useDominantResourceCalculator) {
        when(cs.getResourceCalculator()).thenReturn(new DominantResourceCalculator());
    }
    mockNodeLabelsManager(labelsConfig);
    mockSchedulerNodes(nodesConfig);
    for (NodeId nodeId : nodeIdToSchedulerNodes.keySet()) {
        when(cs.getSchedulerNode(nodeId)).thenReturn(nodeIdToSchedulerNodes.get(nodeId));
    }
    List<FiCaSchedulerNode> allNodes = new ArrayList<>(nodeIdToSchedulerNodes.values());
    when(cs.getAllNodes()).thenReturn(allNodes);
    ParentQueue root = mockQueueHierarchy(queuesConfig);
    when(cs.getRootQueue()).thenReturn(root);
    when(cs.getClusterResource()).thenReturn(clusterResource);
    mockApplications(appsConfig);
    policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock);
}

Example 62 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method markContainerForNonKillable.

private void markContainerForNonKillable(RMContainer nonKillableContainer) {
    try {
        writeLock.lock();
        if (LOG.isDebugEnabled()) {
            LOG.debug(SchedulerEventType.MARK_CONTAINER_FOR_NONKILLABLE + ": container" + nonKillableContainer.toString());
        }
        FiCaSchedulerNode node = (FiCaSchedulerNode) getSchedulerNode(nonKillableContainer.getAllocatedNode());
        FiCaSchedulerApp application = getCurrentAttemptForContainer(nonKillableContainer.getContainerId());
        node.markContainerToNonKillable(nonKillableContainer.getContainerId());
        // Get the application for the finished container
        if (null != application) {
            String leafQueueName = application.getCSLeafQueue().getQueueName();
            getPreemptionManager().removeKillableContainer(new KillableContainer(nonKillableContainer, node.getPartition(), leafQueueName));
        }
    } finally {
        writeLock.unlock();
    }
}

Example 63 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method getSchedulerContainer.

private SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> getSchedulerContainer(RMContainer rmContainer, boolean allocated) {
    if (null == rmContainer) {
        return null;
    }
    FiCaSchedulerApp app = getApplicationAttempt(rmContainer.getApplicationAttemptId());
    if (null == app) {
        return null;
    }
    NodeId nodeId;
    // Get nodeId
    if (rmContainer.getState() == RMContainerState.RESERVED) {
        nodeId = rmContainer.getReservedNode();
    } else {
        nodeId = rmContainer.getNodeId();
    }
    FiCaSchedulerNode node = getNode(nodeId);
    if (null == node) {
        return null;
    }
    return new SchedulerContainer<>(app, node, rmContainer, // get updated before submitting the commit
    node.getPartition(), allocated);
}

Example 64 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method removeNode.

private void removeNode(RMNode nodeInfo) {
    try {
        writeLock.lock();
        // update this node to node label manager
        if (labelManager != null) {
            labelManager.deactivateNode(nodeInfo.getNodeID());
        }
        NodeId nodeId = nodeInfo.getNodeID();
        FiCaSchedulerNode node = nodeTracker.getNode(nodeId);
        if (node == null) {
            LOG.error("Attempting to remove non-existent node " + nodeId);
            return;
        }
        // Remove running containers
        List<RMContainer> runningContainers = node.getCopiedListOfRunningContainers();
        for (RMContainer container : runningContainers) {
            super.completedContainer(container, SchedulerUtils.createAbnormalContainerStatus(container.getContainerId(), SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL);
        }
        // Remove reservations, if any
        RMContainer reservedContainer = node.getReservedContainer();
        if (reservedContainer != null) {
            super.completedContainer(reservedContainer, SchedulerUtils.createAbnormalContainerStatus(reservedContainer.getContainerId(), SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL);
        }
        nodeTracker.removeNode(nodeId);
        Resource clusterResource = getClusterResource();
        getRootQueue().updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
        int numNodes = nodeTracker.nodeCount();
        if (scheduleAsynchronously && numNodes == 0) {
            for (AsyncScheduleThread t : asyncSchedulerThreads) {
                t.suspendSchedule();
            }
        }
        LOG.info("Removed node " + nodeInfo.getNodeAddress() + " clusterResource: " + getClusterResource());
    } finally {
        writeLock.unlock();
    }
}

Example 65 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class CapacityScheduler method moveReservedContainer.

/**
   * Try to move a reserved container to a targetNode.
   * If the targetNode is reserved by another application (other than this one).
   * The previous reservation will be cancelled.
   *
   * @param toBeMovedContainer reserved container will be moved
   * @param targetNode targetNode
   * @return true if move succeeded. Return false if the targetNode is reserved by
   *         a different container or move failed because of any other reasons.
   */
public boolean moveReservedContainer(RMContainer toBeMovedContainer, FiCaSchedulerNode targetNode) {
    try {
        writeLock.lock();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Trying to move container=" + toBeMovedContainer + " to node=" + targetNode.getNodeID());
        }
        FiCaSchedulerNode sourceNode = getNode(toBeMovedContainer.getNodeId());
        if (null == sourceNode) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Failed to move reservation, cannot find source node=" + toBeMovedContainer.getNodeId());
            }
            return false;
        }
        // Target node updated?
        if (getNode(targetNode.getNodeID()) != targetNode) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Failed to move reservation, node updated or removed, moving " + "cancelled.");
            }
            return false;
        }
        // Target node's reservation status changed?
        if (targetNode.getReservedContainer() != null) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Target node's reservation status changed, moving cancelled.");
            }
            return false;
        }
        FiCaSchedulerApp app = getApplicationAttempt(toBeMovedContainer.getApplicationAttemptId());
        if (null == app) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Cannot find to-be-moved container's application=" + toBeMovedContainer.getApplicationAttemptId());
            }
            return false;
        }
        // finally, move the reserved container
        return app.moveReservation(toBeMovedContainer, sourceNode, targetNode);
    } finally {
        writeLock.unlock();
    }
}