Examples with FiCaSchedulerNode org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode used on opensource projects

Example 76 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class LeafQueue method accept.

@Override
public boolean accept(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
    ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation = request.getFirstAllocatedOrReservedContainer();
    SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer = allocation.getAllocatedOrReservedContainer();
    // Do not check limits when allocation from a reserved container
    if (allocation.getAllocateFromReservedContainer() == null) {
        try {
            readLock.lock();
            FiCaSchedulerApp app = schedulerContainer.getSchedulerApplicationAttempt();
            String username = app.getUser();
            String p = schedulerContainer.getNodePartition();
            // check user-limit
            Resource userLimit = computeUserLimitAndSetHeadroom(app, cluster, p, allocation.getSchedulingMode());
            // Deduct resources that we can release
            Resource usedResource = Resources.clone(getUser(username).getUsed(p));
            Resources.subtractFrom(usedResource, request.getTotalReleasedResource());
            if (Resources.greaterThan(resourceCalculator, cluster, usedResource, userLimit)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Used resource=" + usedResource + " exceeded user-limit=" + userLimit);
                }
                return false;
            }
        } finally {
            readLock.unlock();
        }
    }
    return super.accept(cluster, request);
}

Example 77 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class LeafQueue method detachContainer.

@Override
public void detachContainer(Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer) {
    if (application != null && rmContainer != null && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
        FiCaSchedulerNode node = scheduler.getNode(rmContainer.getContainer().getNodeId());
        releaseResource(clusterResource, application, rmContainer.getContainer().getResource(), node.getPartition(), rmContainer);
        LOG.info("movedContainer" + " container=" + rmContainer.getContainer() + " resource=" + rmContainer.getContainer().getResource() + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" + clusterResource);
        // Inform the parent queue
        getParent().detachContainer(clusterResource, application, rmContainer);
    }
}

Example 78 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method mockSchedulerNodes.

/**
   * Format is:
   * host1=partition[ res=resource];
   * host2=partition[ res=resource];
   */
private void mockSchedulerNodes(String schedulerNodesConfigStr) throws IOException {
    String[] nodesConfigStrArray = schedulerNodesConfigStr.split(";");
    for (String p : nodesConfigStrArray) {
        String[] arr = p.split(" ");
        NodeId nodeId = NodeId.newInstance(arr[0].substring(0, arr[0].indexOf("=")), 1);
        String partition = arr[0].substring(arr[0].indexOf("=") + 1, arr[0].length());
        FiCaSchedulerNode sn = mock(FiCaSchedulerNode.class);
        when(sn.getNodeID()).thenReturn(nodeId);
        when(sn.getPartition()).thenReturn(partition);
        Resource totalRes = Resources.createResource(0);
        if (arr.length > 1) {
            String res = arr[1];
            if (res.contains("res=")) {
                String resSring = res.substring(res.indexOf("res=") + "res=".length());
                totalRes = parseResourceFromString(resSring);
            }
        }
        when(sn.getTotalResource()).thenReturn(totalRes);
        when(sn.getUnallocatedResource()).thenReturn(Resources.clone(totalRes));
        // TODO, add settings of killable resources when necessary
        when(sn.getTotalKillableResources()).thenReturn(Resources.none());
        List<RMContainer> liveContainers = new ArrayList<>();
        when(sn.getCopiedListOfRunningContainers()).thenReturn(liveContainers);
        nodeIdToSchedulerNodes.put(nodeId, sn);
        LOG.debug("add scheduler node, id=" + nodeId + ", partition=" + partition);
    }
}

Example 79 with FiCaSchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method buildEnv.

public void buildEnv(String labelsConfig, String nodesConfig, String queuesConfig, String appsConfig, boolean useDominantResourceCalculator) throws IOException {
    if (useDominantResourceCalculator) {
        when(cs.getResourceCalculator()).thenReturn(new DominantResourceCalculator());
    }
    mockNodeLabelsManager(labelsConfig);
    mockSchedulerNodes(nodesConfig);
    for (NodeId nodeId : nodeIdToSchedulerNodes.keySet()) {
        when(cs.getSchedulerNode(nodeId)).thenReturn(nodeIdToSchedulerNodes.get(nodeId));
    }
    List<FiCaSchedulerNode> allNodes = new ArrayList<>(nodeIdToSchedulerNodes.values());
    when(cs.getAllNodes()).thenReturn(allNodes);
    ParentQueue root = mockQueueHierarchy(queuesConfig);
    when(cs.getRootQueue()).thenReturn(root);
    when(cs.getClusterResource()).thenReturn(clusterResource);
    mockApplications(appsConfig);
    policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock);
}