Examples with CSQueue org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue used on opensource projects

Example 16 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class SLSCapacityScheduler method initQueueMetrics.

private void initQueueMetrics(CSQueue queue) {
    if (queue instanceof LeafQueue) {
        SortedMap<String, Counter> counterMap = metrics.getCounters();
        String queueName = queue.getQueueName();
        String[] names = new String[] { QUEUE_COUNTER_PREFIX + queueName + ".pending.memory", QUEUE_COUNTER_PREFIX + queueName + ".pending.cores", QUEUE_COUNTER_PREFIX + queueName + ".allocated.memory", QUEUE_COUNTER_PREFIX + queueName + ".allocated.cores" };
        for (int i = names.length - 1; i >= 0; i--) {
            if (!counterMap.containsKey(names[i])) {
                metrics.counter(names[i]);
                counterMap = metrics.getCounters();
            }
        }
        queueLock.lock();
        try {
            if (!schedulerMetrics.isTracked(queueName)) {
                schedulerMetrics.trackQueue(queueName);
            }
        } finally {
            queueLock.unlock();
        }
        return;
    }
    for (CSQueue child : queue.getChildQueues()) {
        initQueueMetrics(child);
    }
}

Example 17 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicy method cloneQueues.

/**
   * This method walks a tree of CSQueue and clones the portion of the state
   * relevant for preemption in TempQueue(s). It also maintains a pointer to
   * the leaves. Finally it aggregates pending resources in each queue and rolls
   * it up to higher levels.
   *
   * @param curQueue current queue which I'm looking at now
   * @param partitionResource the total amount of resources in the cluster
   * @return the root of the cloned queue hierarchy
   */
private TempQueuePerPartition cloneQueues(CSQueue curQueue, Resource partitionResource, String partitionToLookAt) {
    TempQueuePerPartition ret;
    ReadLock readLock = curQueue.getReadLock();
    try {
        // Acquire a read lock from Parent/LeafQueue.
        readLock.lock();
        String queueName = curQueue.getQueueName();
        QueueCapacities qc = curQueue.getQueueCapacities();
        float absCap = qc.getAbsoluteCapacity(partitionToLookAt);
        float absMaxCap = qc.getAbsoluteMaximumCapacity(partitionToLookAt);
        boolean preemptionDisabled = curQueue.getPreemptionDisabled();
        Resource current = Resources.clone(curQueue.getQueueResourceUsage().getUsed(partitionToLookAt));
        Resource killable = Resources.none();
        Resource reserved = Resources.clone(curQueue.getQueueResourceUsage().getReserved(partitionToLookAt));
        if (null != preemptableQueues.get(queueName)) {
            killable = Resources.clone(preemptableQueues.get(queueName).getKillableResource(partitionToLookAt));
        }
        // could more than specified maxCapacity
        try {
            if (!scheduler.getRMContext().getNodeLabelManager().isExclusiveNodeLabel(partitionToLookAt)) {
                absMaxCap = 1.0f;
            }
        } catch (IOException e) {
        // This may cause by partition removed when running capacity monitor,
        // just ignore the error, this will be corrected when doing next check.
        }
        ret = new TempQueuePerPartition(queueName, current, preemptionDisabled, partitionToLookAt, killable, absCap, absMaxCap, partitionResource, reserved, curQueue);
        if (curQueue instanceof ParentQueue) {
            String configuredOrderingPolicy = ((ParentQueue) curQueue).getQueueOrderingPolicy().getConfigName();
            // Recursively add children
            for (CSQueue c : curQueue.getChildQueues()) {
                TempQueuePerPartition subq = cloneQueues(c, partitionResource, partitionToLookAt);
                // If we respect priority
                if (StringUtils.equals(CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY, configuredOrderingPolicy)) {
                    subq.relativePriority = c.getPriority().getPriority();
                }
                ret.addChild(subq);
                subq.parent = ret;
            }
        }
    } finally {
        readLock.unlock();
    }
    addTempQueuePartition(ret);
    return ret;
}

Example 18 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicy method editSchedule.

@Override
public synchronized void editSchedule() {
    long startTs = clock.getTime();
    CSQueue root = scheduler.getRootQueue();
    Resource clusterResources = Resources.clone(scheduler.getClusterResource());
    containerBasedPreemptOrKill(root, clusterResources);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Total time used=" + (clock.getTime() - startTs) + " ms.");
    }
}

Example 19 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class CapacitySchedulerPlanFollower method getChildReservationQueues.

@Override
protected List<? extends Queue> getChildReservationQueues(Queue queue) {
    PlanQueue planQueue = (PlanQueue) queue;
    List<CSQueue> childQueues = planQueue.getChildQueues();
    return childQueues;
}

Example 20 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class CapacitySchedulerPlanFollower method getReservationQueueResourceIfExists.

@Override
protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) {
    CSQueue resQueue = cs.getQueue(reservationId.toString());
    Resource reservationResource = null;
    if (resQueue != null) {
        reservationResource = Resources.multiply(cs.getClusterResource(), resQueue.getAbsoluteCapacity());
    }
    return reservationResource;
}