Examples with ParentQueue org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue used on opensource projects

Example 1 with ParentQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method mockQueueHierarchy.

/**
   * Format is:
   * <pre>
   * root (<partition-name-1>=[guaranteed max used pending (reserved)],<partition-name-2>=..);
   * -A(...);
   * --A1(...);
   * --A2(...);
   * -B...
   * </pre>
   * ";" splits queues, and there should no empty lines, no extra spaces
   *
   * For each queue, it has configurations to specify capacities (to each
   * partition), format is:
   * <pre>
   * -<queueName> (<labelName1>=[guaranteed max used pending], \
   *               <labelName2>=[guaranteed max used pending])
   *              {key1=value1,key2=value2};  // Additional configs
   * </pre>
   */
@SuppressWarnings({ "unchecked", "rawtypes" })
private ParentQueue mockQueueHierarchy(String queueExprs) {
    String[] queueExprArray = queueExprs.split(";");
    ParentQueue rootQueue = null;
    for (int idx = 0; idx < queueExprArray.length; idx++) {
        String q = queueExprArray[idx];
        CSQueue queue;
        // Initialize queue
        if (isParent(queueExprArray, idx)) {
            ParentQueue parentQueue = mock(ParentQueue.class);
            queue = parentQueue;
            List<CSQueue> children = new ArrayList<CSQueue>();
            when(parentQueue.getChildQueues()).thenReturn(children);
            QueueOrderingPolicy policy = mock(QueueOrderingPolicy.class);
            when(policy.getConfigName()).thenReturn(CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
            when(parentQueue.getQueueOrderingPolicy()).thenReturn(policy);
        } else {
            LeafQueue leafQueue = mock(LeafQueue.class);
            final TreeSet<FiCaSchedulerApp> apps = new TreeSet<>(new Comparator<FiCaSchedulerApp>() {

                @Override
                public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) {
                    if (a1.getPriority() != null && !a1.getPriority().equals(a2.getPriority())) {
                        return a1.getPriority().compareTo(a2.getPriority());
                    }
                    int res = a1.getApplicationId().compareTo(a2.getApplicationId());
                    return res;
                }
            });
            when(leafQueue.getApplications()).thenReturn(apps);
            when(leafQueue.getAllApplications()).thenReturn(apps);
            OrderingPolicy<FiCaSchedulerApp> so = mock(OrderingPolicy.class);
            when(so.getPreemptionIterator()).thenAnswer(new Answer() {

                public Object answer(InvocationOnMock invocation) {
                    return apps.descendingIterator();
                }
            });
            when(leafQueue.getOrderingPolicy()).thenReturn(so);
            Map<String, TreeSet<RMContainer>> ignorePartitionContainers = new HashMap<>();
            when(leafQueue.getIgnoreExclusivityRMContainers()).thenReturn(ignorePartitionContainers);
            queue = leafQueue;
        }
        ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
        when(queue.getReadLock()).thenReturn(lock.readLock());
        setupQueue(queue, q, queueExprArray, idx);
        if (queue.getQueueName().equals(ROOT)) {
            rootQueue = (ParentQueue) queue;
        }
    }
    return rootQueue;
}

Example 2 with ParentQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method buildPolicy.

ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData, String[][] resData, boolean useDominantResourceCalculator) {
    if (useDominantResourceCalculator) {
        when(mCS.getResourceCalculator()).thenReturn(new DominantResourceCalculator());
    }
    ProportionalCapacityPreemptionPolicy policy = new ProportionalCapacityPreemptionPolicy(rmContext, mCS, mClock);
    clusterResources = leafAbsCapacities(parseResourceDetails(resData[0]), qData[2]);
    ParentQueue mRoot = buildMockRootQueue(rand, resData, qData);
    when(mCS.getRootQueue()).thenReturn(mRoot);
    setResourceAndNodeDetails();
    return policy;
}

Example 3 with ParentQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method buildPolicy.

ProportionalCapacityPreemptionPolicy buildPolicy(int[][] qData) {
    ProportionalCapacityPreemptionPolicy policy = new ProportionalCapacityPreemptionPolicy(rmContext, mCS, mClock);
    clusterResources = Resource.newInstance(leafAbsCapacities(qData[0], qData[7]), 0);
    ParentQueue mRoot = buildMockRootQueue(rand, qData);
    when(mCS.getRootQueue()).thenReturn(mRoot);
    setResourceAndNodeDetails();
    return policy;
}

Example 4 with ParentQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue in project hadoop by apache.

the class TestWorkPreservingRMRestart method testCapacitySchedulerRecovery.

// Test CS recovery with multi-level queues and multi-users:
// 1. setup 2 NMs each with 8GB memory;
// 2. setup 2 level queues: Default -> (QueueA, QueueB)
// 3. User1 submits 2 apps on QueueA
// 4. User2 submits 1 app  on QueueB
// 5. AM and each container has 1GB memory
// 6. Restart RM.
// 7. nm1 re-syncs back containers belong to user1
// 8. nm2 re-syncs back containers belong to user2.
// 9. Assert the parent queue and 2 leaf queues state and the metrics.
// 10. Assert each user's consumption inside the queue.
@Test(timeout = 30000)
public void testCapacitySchedulerRecovery() throws Exception {
    if (getSchedulerType() != SchedulerType.CAPACITY) {
        return;
    }
    conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
    conf.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class.getName());
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfiguration(csConf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(csConf);
    rm1 = new MockRM(csConf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    MockNM nm2 = new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    nm2.registerNode();
    RMApp app1_1 = rm1.submitApp(1024, "app1_1", USER_1, null, A);
    MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
    RMApp app1_2 = rm1.submitApp(1024, "app1_2", USER_1, null, A);
    MockAM am1_2 = MockRM.launchAndRegisterAM(app1_2, rm1, nm2);
    RMApp app2 = rm1.submitApp(1024, "app2", USER_2, null, B);
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2);
    // clear queue metrics
    rm1.clearQueueMetrics(app1_1);
    rm1.clearQueueMetrics(app1_2);
    rm1.clearQueueMetrics(app2);
    csConf.set(PREFIX + "root.Default.QueueB.state", "STOPPED");
    // Re-start RM
    rm2 = new MockRM(csConf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    nm2.setResourceTrackerService(rm2.getResourceTrackerService());
    List<NMContainerStatus> am1_1Containers = createNMContainerStatusForApp(am1_1);
    List<NMContainerStatus> am1_2Containers = createNMContainerStatusForApp(am1_2);
    am1_1Containers.addAll(am1_2Containers);
    nm1.registerNode(am1_1Containers, null);
    List<NMContainerStatus> am2Containers = createNMContainerStatusForApp(am2);
    nm2.registerNode(am2Containers, null);
    // Wait for RM to settle down on recovering containers;
    waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());
    waitForNumContainersToRecover(2, rm2, am1_2.getApplicationAttemptId());
    waitForNumContainersToRecover(2, rm2, am2.getApplicationAttemptId());
    // Calculate each queue's resource usage.
    Resource containerResource = Resource.newInstance(1024, 1);
    Resource nmResource = Resource.newInstance(nm1.getMemory(), nm1.getvCores());
    Resource clusterResource = Resources.multiply(nmResource, 2);
    Resource q1Resource = Resources.multiply(clusterResource, 0.5);
    Resource q2Resource = Resources.multiply(clusterResource, 0.5);
    Resource q1UsedResource = Resources.multiply(containerResource, 4);
    Resource q2UsedResource = Resources.multiply(containerResource, 2);
    Resource totalUsedResource = Resources.add(q1UsedResource, q2UsedResource);
    Resource q1availableResources = Resources.subtract(q1Resource, q1UsedResource);
    Resource q2availableResources = Resources.subtract(q2Resource, q2UsedResource);
    Resource totalAvailableResource = Resources.add(q1availableResources, q2availableResources);
    Map<ApplicationId, SchedulerApplication> schedulerApps = ((AbstractYarnScheduler) rm2.getResourceScheduler()).getSchedulerApplications();
    SchedulerApplication schedulerApp1_1 = schedulerApps.get(app1_1.getApplicationId());
    // assert queue A state.
    checkCSLeafQueue(rm2, schedulerApp1_1, clusterResource, q1Resource, q1UsedResource, 4);
    QueueMetrics queue1Metrics = schedulerApp1_1.getQueue().getMetrics();
    assertMetrics(queue1Metrics, 2, 0, 2, 0, 4, q1availableResources.getMemorySize(), q1availableResources.getVirtualCores(), q1UsedResource.getMemorySize(), q1UsedResource.getVirtualCores());
    // assert queue B state.
    SchedulerApplication schedulerApp2 = schedulerApps.get(app2.getApplicationId());
    checkCSLeafQueue(rm2, schedulerApp2, clusterResource, q2Resource, q2UsedResource, 2);
    QueueMetrics queue2Metrics = schedulerApp2.getQueue().getMetrics();
    assertMetrics(queue2Metrics, 1, 0, 1, 0, 2, q2availableResources.getMemorySize(), q2availableResources.getVirtualCores(), q2UsedResource.getMemorySize(), q2UsedResource.getVirtualCores());
    // assert parent queue state.
    LeafQueue leafQueue = (LeafQueue) schedulerApp2.getQueue();
    ParentQueue parentQueue = (ParentQueue) leafQueue.getParent();
    checkParentQueue(parentQueue, 6, totalUsedResource, (float) 6 / 16, (float) 6 / 16);
    assertMetrics(parentQueue.getMetrics(), 3, 0, 3, 0, 6, totalAvailableResource.getMemorySize(), totalAvailableResource.getVirtualCores(), totalUsedResource.getMemorySize(), totalUsedResource.getVirtualCores());
}

Example 5 with ParentQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method getParentQueue.

private ParentQueue getParentQueue(String[] queueExprArray, int idx, int myLevel) {
    idx--;
    while (idx >= 0) {
        int level = getLevel(queueExprArray[idx]);
        if (level < myLevel) {
            String parentQueuName = getQueueName(queueExprArray[idx]);
            return (ParentQueue) nameToCSQueues.get(parentQueuName);
        }
        idx--;
    }
    return null;
}