Examples with CSQueue org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue used on opensource projects

Example 11 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class TestPriorityUtilizationQueueOrderingPolicy method verifyOrder.

private void verifyOrder(QueueOrderingPolicy orderingPolicy, String partition, String[] expectedOrder) {
    Iterator<CSQueue> iter = orderingPolicy.getAssignmentIterator(partition);
    int i = 0;
    while (iter.hasNext()) {
        CSQueue q = iter.next();
        Assert.assertEquals(expectedOrder[i], q.getQueueName());
        i++;
    }
    assert i == expectedOrder.length;
}

Example 12 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class PreemptionManager method refreshQueues.

public void refreshQueues(CSQueue parent, CSQueue current) {
    try {
        writeLock.lock();
        PreemptableQueue parentEntity = null;
        if (parent != null) {
            parentEntity = entities.get(parent.getQueueName());
        }
        if (!entities.containsKey(current.getQueueName())) {
            entities.put(current.getQueueName(), new PreemptableQueue(parentEntity));
        }
        if (current.getChildQueues() != null) {
            for (CSQueue child : current.getChildQueues()) {
                refreshQueues(current, child);
            }
        }
    } finally {
        writeLock.unlock();
    }
}

Example 13 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class CapacitySchedulerInfo method getQueues.

protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
    CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList();
    // JAXB marashalling leads to situation where the "type" field injected
    // for JSON changes from string to array depending on order of printing
    // Issue gets fixed if all the leaf queues are marshalled before the
    // non-leaf queues. See YARN-4785 for more details.
    List<CSQueue> childQueues = new ArrayList<>();
    List<CSQueue> childLeafQueues = new ArrayList<>();
    List<CSQueue> childNonLeafQueues = new ArrayList<>();
    for (CSQueue queue : parent.getChildQueues()) {
        if (queue instanceof LeafQueue) {
            childLeafQueues.add(queue);
        } else {
            childNonLeafQueues.add(queue);
        }
    }
    childQueues.addAll(childLeafQueues);
    childQueues.addAll(childNonLeafQueues);
    for (CSQueue queue : childQueues) {
        CapacitySchedulerQueueInfo info;
        if (queue instanceof LeafQueue) {
            info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue);
        } else {
            info = new CapacitySchedulerQueueInfo(queue);
            info.queues = getQueues(queue);
        }
        queuesInfo.addToQueueInfoList(info);
    }
    return queuesInfo;
}

Example 14 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method mockParentQueue.

ParentQueue mockParentQueue(ParentQueue p, int subqueues, Deque<ParentQueue> pqs) {
    ParentQueue pq = mock(ParentQueue.class);
    List<CSQueue> cqs = new ArrayList<CSQueue>();
    when(pq.getChildQueues()).thenReturn(cqs);
    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
    when(pq.getReadLock()).thenReturn(lock.readLock());
    // Ordering policy
    QueueOrderingPolicy policy = mock(QueueOrderingPolicy.class);
    when(policy.getConfigName()).thenReturn(CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
    when(pq.getQueueOrderingPolicy()).thenReturn(policy);
    when(pq.getPriority()).thenReturn(Priority.newInstance(0));
    for (int i = 0; i < subqueues; ++i) {
        pqs.add(pq);
    }
    if (p != null) {
        p.getChildQueues().add(pq);
    }
    return pq;
}

Example 15 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method mockNested.

ParentQueue mockNested(Resource[] abs, int[] maxCap, Resource[] used, Resource[] pending, Resource[] reserved, int[] apps, Resource[] gran, int[] queues) {
    ResourceCalculator rc = mCS.getResourceCalculator();
    Resource tot = leafAbsCapacities(abs, queues);
    Deque<ParentQueue> pqs = new LinkedList<ParentQueue>();
    ParentQueue root = mockParentQueue(null, queues[0], pqs);
    ResourceUsage resUsage = new ResourceUsage();
    resUsage.setUsed(used[0]);
    resUsage.setReserved(reserved[0]);
    when(root.getQueueName()).thenReturn(CapacitySchedulerConfiguration.ROOT);
    when(root.getAbsoluteUsedCapacity()).thenReturn(Resources.divide(rc, tot, used[0], tot));
    when(root.getAbsoluteCapacity()).thenReturn(Resources.divide(rc, tot, abs[0], tot));
    when(root.getAbsoluteMaximumCapacity()).thenReturn(maxCap[0] / (float) tot.getMemorySize());
    when(root.getQueueResourceUsage()).thenReturn(resUsage);
    QueueCapacities rootQc = new QueueCapacities(true);
    rootQc.setAbsoluteUsedCapacity(Resources.divide(rc, tot, used[0], tot));
    rootQc.setAbsoluteCapacity(Resources.divide(rc, tot, abs[0], tot));
    rootQc.setAbsoluteMaximumCapacity(maxCap[0] / (float) tot.getMemorySize());
    when(root.getQueueCapacities()).thenReturn(rootQc);
    when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT);
    boolean preemptionDisabled = mockPreemptionStatus("root");
    when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
    for (int i = 1; i < queues.length; ++i) {
        final CSQueue q;
        final ParentQueue p = pqs.removeLast();
        final String queueName = "queue" + ((char) ('A' + i - 1));
        if (queues[i] > 0) {
            q = mockParentQueue(p, queues[i], pqs);
            ResourceUsage resUsagePerQueue = new ResourceUsage();
            resUsagePerQueue.setUsed(used[i]);
            resUsagePerQueue.setReserved(reserved[i]);
            when(q.getQueueResourceUsage()).thenReturn(resUsagePerQueue);
        } else {
            q = mockLeafQueue(p, tot, i, abs, used, pending, reserved, apps, gran);
        }
        when(q.getParent()).thenReturn(p);
        when(q.getQueueName()).thenReturn(queueName);
        when(q.getAbsoluteUsedCapacity()).thenReturn(Resources.divide(rc, tot, used[i], tot));
        when(q.getAbsoluteCapacity()).thenReturn(Resources.divide(rc, tot, abs[i], tot));
        when(q.getAbsoluteMaximumCapacity()).thenReturn(maxCap[i] / (float) tot.getMemorySize());
        // We need to make these fields to QueueCapacities
        QueueCapacities qc = new QueueCapacities(false);
        qc.setAbsoluteUsedCapacity(Resources.divide(rc, tot, used[i], tot));
        qc.setAbsoluteCapacity(Resources.divide(rc, tot, abs[i], tot));
        qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize());
        when(q.getQueueCapacities()).thenReturn(qc);
        String parentPathName = p.getQueuePath();
        parentPathName = (parentPathName == null) ? "root" : parentPathName;
        String queuePathName = (parentPathName + "." + queueName).replace("/", "root");
        when(q.getQueuePath()).thenReturn(queuePathName);
        preemptionDisabled = mockPreemptionStatus(queuePathName);
        when(q.getPreemptionDisabled()).thenReturn(preemptionDisabled);
    }
    assert 0 == pqs.size();
    return root;
}