Examples with LeafQueue org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue used on opensource projects

Example 11 with LeafQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue in project hadoop by apache.

the class TestFairOrderingPolicy method testSizeBasedWeightNotAffectAppActivation.

@Test
public void testSizeBasedWeightNotAffectAppActivation() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    // Define top-level queues
    String queuePath = CapacitySchedulerConfiguration.ROOT + ".default";
    csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY);
    csConf.setOrderingPolicyParameter(queuePath, FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true");
    csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f);
    // inject node label manager
    MockRM rm = new MockRM(csConf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    // Get LeafQueue
    LeafQueue lq = (LeafQueue) cs.getQueue("default");
    OrderingPolicy<FiCaSchedulerApp> policy = lq.getOrderingPolicy();
    Assert.assertTrue(policy instanceof FairOrderingPolicy);
    Assert.assertTrue(((FairOrderingPolicy<FiCaSchedulerApp>) policy).getSizeBasedWeight());
    rm.registerNode("h1:1234", 10 * GB);
    // Submit 4 apps
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    Assert.assertEquals(1, lq.getNumActiveApplications());
    Assert.assertEquals(3, lq.getNumPendingApplications());
    // Try allocate once, #active-apps and #pending-apps should be still correct
    cs.handle(new NodeUpdateSchedulerEvent(rm.getRMContext().getRMNodes().get(NodeId.newInstance("h1", 1234))));
    Assert.assertEquals(1, lq.getNumActiveApplications());
    Assert.assertEquals(3, lq.getNumPendingApplications());
}

Example 12 with LeafQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicyMockFramework method testBuilder.

@Test
public void testBuilder() throws Exception {
    /**
     * Test of test, make sure we build expected mock schedulable objects
     */
    String labelsConfig = // default partition
    "=200,true;" + // partition=red
    "red=100,false;" + // partition=blue
    "blue=200,true";
    String nodesConfig = // n1 has partition=red
    "n1=red;" + // n2 has partition=blue
    "n2=blue;" + // n3 doesn't have partition
    "n3=";
    String queuesConfig = // guaranteed,max,used,pending
    //root
    "root(=[200 200 100 100],red=[100 100 100 100],blue=[200 200 200 200]);" + // a
    "-a(=[100 200 100 100],red=[0 0 0 0],blue=[200 200 200 200]);" + // a1
    "--a1(=[50 100 50 100],red=[0 0 0 0],blue=[100 200 200 0]);" + // a2
    "--a2(=[50 200 50 0],red=[0 0 0 0],blue=[100 200 0 200]){priority=2};" + "-b(=[100 200 0 0],red=[100 100 100 100],blue=[0 0 0 0]){priority=1,disable_preemption=true}";
    String appsConfig = // app1 in a1, , 50 in n2 (reserved), 50 in n2 (allocated)
    // app1 in a1
    "a1\t" + // 50 * default in n3
    "(1,1,n3,red,50,false);" + // app2 in a1
    "a1\t" + // 50 * ignore-exclusivity (reserved),
    "(2,1,n2,,50,true)(2,1,n2,,50,false)" + // 50 * ignore-exclusivity (allocated)
    "(2,1,n2,blue,50,true)(2,1,n2,blue,50,true);" + // 50 in n2 (allocated)
    "a2\t" + // app3 in a2
    "(1,1,n3,red,50,false);" + // app4 in b
    "b\t" + "(1,1,n1,red,100,false);";
    buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
    // Check queues:
    // root
    checkAbsCapacities(cs.getQueue("root"), "", 1f, 1f, 0.5f);
    checkPendingResource(cs.getQueue("root"), "", 100);
    checkAbsCapacities(cs.getQueue("root"), "red", 1f, 1f, 1f);
    checkPendingResource(cs.getQueue("root"), "red", 100);
    checkAbsCapacities(cs.getQueue("root"), "blue", 1f, 1f, 1f);
    checkPendingResource(cs.getQueue("root"), "blue", 200);
    // default
    checkPriority(cs.getQueue("root"), 0);
    // a
    checkAbsCapacities(cs.getQueue("a"), "", 0.5f, 1f, 0.5f);
    checkPendingResource(cs.getQueue("a"), "", 100);
    checkAbsCapacities(cs.getQueue("a"), "red", 0f, 0f, 0f);
    checkPendingResource(cs.getQueue("a"), "red", 0);
    checkAbsCapacities(cs.getQueue("a"), "blue", 1f, 1f, 1f);
    checkPendingResource(cs.getQueue("a"), "blue", 200);
    // default
    checkPriority(cs.getQueue("a"), 0);
    // a1
    checkAbsCapacities(cs.getQueue("a1"), "", 0.25f, 0.5f, 0.25f);
    checkPendingResource(cs.getQueue("a1"), "", 100);
    checkAbsCapacities(cs.getQueue("a1"), "red", 0f, 0f, 0f);
    checkPendingResource(cs.getQueue("a1"), "red", 0);
    checkAbsCapacities(cs.getQueue("a1"), "blue", 0.5f, 1f, 1f);
    checkPendingResource(cs.getQueue("a1"), "blue", 0);
    // default
    checkPriority(cs.getQueue("a1"), 0);
    // a2
    checkAbsCapacities(cs.getQueue("a2"), "", 0.25f, 1f, 0.25f);
    checkPendingResource(cs.getQueue("a2"), "", 0);
    checkAbsCapacities(cs.getQueue("a2"), "red", 0f, 0f, 0f);
    checkPendingResource(cs.getQueue("a2"), "red", 0);
    checkAbsCapacities(cs.getQueue("a2"), "blue", 0.5f, 1f, 0f);
    checkPendingResource(cs.getQueue("a2"), "blue", 200);
    checkPriority(cs.getQueue("a2"), 2);
    Assert.assertFalse(cs.getQueue("a2").getPreemptionDisabled());
    // b
    checkAbsCapacities(cs.getQueue("b"), "", 0.5f, 1f, 0f);
    checkPendingResource(cs.getQueue("b"), "", 0);
    checkAbsCapacities(cs.getQueue("b"), "red", 1f, 1f, 1f);
    checkPendingResource(cs.getQueue("b"), "red", 100);
    checkAbsCapacities(cs.getQueue("b"), "blue", 0f, 0f, 0f);
    checkPendingResource(cs.getQueue("b"), "blue", 0);
    checkPriority(cs.getQueue("b"), 1);
    Assert.assertTrue(cs.getQueue("b").getPreemptionDisabled());
    // Check ignored partitioned containers in queue
    Assert.assertEquals(100, ((LeafQueue) cs.getQueue("a1")).getIgnoreExclusivityRMContainers().get("blue").size());
    // Check applications
    Assert.assertEquals(2, ((LeafQueue) cs.getQueue("a1")).getApplications().size());
    Assert.assertEquals(1, ((LeafQueue) cs.getQueue("a2")).getApplications().size());
    Assert.assertEquals(1, ((LeafQueue) cs.getQueue("b")).getApplications().size());
    // Check #containers
    FiCaSchedulerApp app1 = getApp("a1", 1);
    FiCaSchedulerApp app2 = getApp("a1", 2);
    FiCaSchedulerApp app3 = getApp("a2", 3);
    FiCaSchedulerApp app4 = getApp("b", 4);
    Assert.assertEquals(50, app1.getLiveContainers().size());
    checkContainerNodesInApp(app1, 50, "n3");
    Assert.assertEquals(50, app2.getLiveContainers().size());
    Assert.assertEquals(150, app2.getReservedContainers().size());
    checkContainerNodesInApp(app2, 200, "n2");
    Assert.assertEquals(50, app3.getLiveContainers().size());
    checkContainerNodesInApp(app3, 50, "n3");
    Assert.assertEquals(100, app4.getLiveContainers().size());
    checkContainerNodesInApp(app4, 100, "n1");
}

Example 13 with LeafQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method mockContainers.

private void mockContainers(String containersConfig, FiCaSchedulerApp app, ApplicationAttemptId attemptId, String queueName, List<RMContainer> reservedContainers, List<RMContainer> liveContainers) {
    int containerId = 1;
    int start = containersConfig.indexOf("=") + 1;
    int end = -1;
    Resource used = Resource.newInstance(0, 0);
    Resource pending = Resource.newInstance(0, 0);
    Priority pri = Priority.newInstance(0);
    while (start < containersConfig.length()) {
        while (start < containersConfig.length() && containersConfig.charAt(start) != '(') {
            start++;
        }
        if (start >= containersConfig.length()) {
            throw new IllegalArgumentException("Error containers specification, line=" + containersConfig);
        }
        end = start + 1;
        while (end < containersConfig.length() && containersConfig.charAt(end) != ')') {
            end++;
        }
        if (end >= containersConfig.length()) {
            throw new IllegalArgumentException("Error containers specification, line=" + containersConfig);
        }
        // now we found start/end, get container values
        String[] values = containersConfig.substring(start + 1, end).split(",");
        if (values.length < 6 || values.length > 8) {
            throw new IllegalArgumentException("Format to define container is:" + "(priority,resource,host,expression,repeat,reserved, pending)");
        }
        pri.setPriority(Integer.valueOf(values[0]));
        Resource res = parseResourceFromString(values[1]);
        NodeId host = NodeId.newInstance(values[2], 1);
        String label = values[3];
        String userName = "user";
        int repeat = Integer.valueOf(values[4]);
        boolean reserved = Boolean.valueOf(values[5]);
        if (values.length >= 7) {
            Resources.addTo(pending, parseResourceFromString(values[6]));
        }
        if (values.length == 8) {
            userName = values[7];
        }
        for (int i = 0; i < repeat; i++) {
            Container c = mock(Container.class);
            Resources.addTo(used, res);
            when(c.getResource()).thenReturn(res);
            when(c.getPriority()).thenReturn(pri);
            SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(c);
            RMContainerImpl rmc = mock(RMContainerImpl.class);
            when(rmc.getAllocatedSchedulerKey()).thenReturn(sk);
            when(rmc.getAllocatedNode()).thenReturn(host);
            when(rmc.getNodeLabelExpression()).thenReturn(label);
            when(rmc.getAllocatedResource()).thenReturn(res);
            when(rmc.getContainer()).thenReturn(c);
            when(rmc.getApplicationAttemptId()).thenReturn(attemptId);
            when(rmc.getQueueName()).thenReturn(queueName);
            final ContainerId cId = ContainerId.newContainerId(attemptId, containerId);
            when(rmc.getContainerId()).thenReturn(cId);
            doAnswer(new Answer<Integer>() {

                @Override
                public Integer answer(InvocationOnMock invocation) throws Throwable {
                    return cId.compareTo(((RMContainer) invocation.getArguments()[0]).getContainerId());
                }
            }).when(rmc).compareTo(any(RMContainer.class));
            if (containerId == 1) {
                when(rmc.isAMContainer()).thenReturn(true);
                when(app.getAMResource(label)).thenReturn(res);
            }
            if (reserved) {
                reservedContainers.add(rmc);
                when(rmc.getReservedResource()).thenReturn(res);
            } else {
                liveContainers.add(rmc);
            }
            // Add container to scheduler-node
            addContainerToSchedulerNode(host, rmc, reserved);
            // If this is a non-exclusive allocation
            String partition = null;
            if (label.isEmpty() && !(partition = nodeIdToSchedulerNodes.get(host).getPartition()).isEmpty()) {
                LeafQueue queue = (LeafQueue) nameToCSQueues.get(queueName);
                Map<String, TreeSet<RMContainer>> ignoreExclusivityContainers = queue.getIgnoreExclusivityRMContainers();
                if (!ignoreExclusivityContainers.containsKey(partition)) {
                    ignoreExclusivityContainers.put(partition, new TreeSet<RMContainer>());
                }
                ignoreExclusivityContainers.get(partition).add(rmc);
            }
            LOG.debug("add container to app=" + attemptId + " res=" + res + " node=" + host + " nodeLabelExpression=" + label + " partition=" + partition);
            containerId++;
        }
        // Some more app specific aggregated data can be better filled here.
        when(app.getPriority()).thenReturn(pri);
        when(app.getUser()).thenReturn(userName);
        when(app.getCurrentConsumption()).thenReturn(used);
        when(app.getCurrentReservation()).thenReturn(Resources.createResource(0, 0));
        Map<String, Resource> pendingForDefaultPartition = new HashMap<String, Resource>();
        // Add for default partition for now.
        pendingForDefaultPartition.put(label, pending);
        when(app.getTotalPendingRequestsPerPartition()).thenReturn(pendingForDefaultPartition);
        // need to set pending resource in resource usage as well
        ResourceUsage ru = new ResourceUsage();
        ru.setUsed(label, used);
        when(app.getAppAttemptResourceUsage()).thenReturn(ru);
        start = end + 1;
    }
}

Example 14 with LeafQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method setupQueue.

private void setupQueue(CSQueue queue, String q, String[] queueExprArray, int idx) {
    LOG.debug("*** Setup queue, source=" + q);
    String queuePath = null;
    int myLevel = getLevel(q);
    if (0 == myLevel) {
        // It's root
        when(queue.getQueueName()).thenReturn(ROOT);
        queuePath = ROOT;
    }
    String queueName = getQueueName(q);
    when(queue.getQueueName()).thenReturn(queueName);
    // Setup parent queue, and add myself to parentQueue.children-list
    ParentQueue parentQueue = getParentQueue(queueExprArray, idx, myLevel);
    if (null != parentQueue) {
        when(queue.getParent()).thenReturn(parentQueue);
        parentQueue.getChildQueues().add(queue);
        // Setup my path
        queuePath = parentQueue.getQueuePath() + "." + queueName;
    }
    when(queue.getQueuePath()).thenReturn(queuePath);
    QueueCapacities qc = new QueueCapacities(0 == myLevel);
    ResourceUsage ru = new ResourceUsage();
    when(queue.getQueueCapacities()).thenReturn(qc);
    when(queue.getQueueResourceUsage()).thenReturn(ru);
    LOG.debug("Setup queue, name=" + queue.getQueueName() + " path=" + queue.getQueuePath());
    LOG.debug("Parent=" + (parentQueue == null ? "null" : parentQueue.getQueueName()));
    // Setup other fields like used resource, guaranteed resource, etc.
    String capacitySettingStr = q.substring(q.indexOf("(") + 1, q.indexOf(")"));
    for (String s : capacitySettingStr.split(",")) {
        String partitionName = s.substring(0, s.indexOf("="));
        String[] values = s.substring(s.indexOf("[") + 1, s.indexOf("]")).split(" ");
        // Add a small epsilon to capacities to avoid truncate when doing
        // Resources.multiply
        float epsilon = 1e-6f;
        Resource totResoucePerPartition = partitionToResource.get(partitionName);
        float absGuaranteed = Resources.divide(rc, totResoucePerPartition, parseResourceFromString(values[0].trim()), totResoucePerPartition) + epsilon;
        float absMax = Resources.divide(rc, totResoucePerPartition, parseResourceFromString(values[1].trim()), totResoucePerPartition) + epsilon;
        float absUsed = Resources.divide(rc, totResoucePerPartition, parseResourceFromString(values[2].trim()), totResoucePerPartition) + epsilon;
        float used = Resources.divide(rc, totResoucePerPartition, parseResourceFromString(values[2].trim()), parseResourceFromString(values[0].trim())) + epsilon;
        Resource pending = parseResourceFromString(values[3].trim());
        qc.setAbsoluteCapacity(partitionName, absGuaranteed);
        qc.setAbsoluteMaximumCapacity(partitionName, absMax);
        qc.setAbsoluteUsedCapacity(partitionName, absUsed);
        qc.setUsedCapacity(partitionName, used);
        when(queue.getUsedCapacity()).thenReturn(used);
        ru.setPending(partitionName, pending);
        // Setup reserved resource if it contained by input config
        Resource reserved = Resources.none();
        if (values.length == 5) {
            reserved = parseResourceFromString(values[4].trim());
            ru.setReserved(partitionName, reserved);
        }
        if (!isParent(queueExprArray, idx)) {
            LeafQueue lq = (LeafQueue) queue;
            when(lq.getTotalPendingResourcesConsideringUserLimit(isA(Resource.class), isA(String.class), eq(false))).thenReturn(pending);
            when(lq.getTotalPendingResourcesConsideringUserLimit(isA(Resource.class), isA(String.class), eq(true))).thenReturn(Resources.subtract(pending, reserved));
        }
        ru.setUsed(partitionName, parseResourceFromString(values[2].trim()));
        LOG.debug("Setup queue=" + queueName + " partition=" + partitionName + " [abs_guaranteed=" + absGuaranteed + ",abs_max=" + absMax + ",abs_used" + absUsed + ",pending_resource=" + pending + ", reserved_resource=" + reserved + "]");
    }
    // Setup preemption disabled
    when(queue.getPreemptionDisabled()).thenReturn(conf.getPreemptionDisabled(queuePath, false));
    // Setup other queue configurations
    Map<String, String> otherConfigs = getOtherConfigurations(queueExprArray[idx]);
    if (otherConfigs.containsKey("priority")) {
        when(queue.getPriority()).thenReturn(Priority.newInstance(Integer.valueOf(otherConfigs.get("priority"))));
    } else {
        // set queue's priority to 0 by default
        when(queue.getPriority()).thenReturn(Priority.newInstance(0));
    }
    // Setup disable preemption of queues
    if (otherConfigs.containsKey("disable_preemption")) {
        when(queue.getPreemptionDisabled()).thenReturn(Boolean.valueOf(otherConfigs.get("disable_preemption")));
    }
    nameToCSQueues.put(queueName, queue);
    when(cs.getQueue(eq(queueName))).thenReturn(queue);
}

Example 15 with LeafQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue in project hadoop by apache.

the class TestWorkPreservingRMRestart method checkCSLeafQueue.

private void checkCSLeafQueue(MockRM rm, SchedulerApplication<SchedulerApplicationAttempt> app, Resource clusterResource, Resource queueResource, Resource usedResource, int numContainers) {
    LeafQueue leafQueue = (LeafQueue) app.getQueue();
    // assert queue used resources.
    assertEquals(usedResource, leafQueue.getUsedResources());
    assertEquals(numContainers, leafQueue.getNumContainers());
    ResourceCalculator calc = ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
    float usedCapacity = Resources.divide(calc, clusterResource, usedResource, queueResource);
    // assert queue used capacity
    assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
    float absoluteUsedCapacity = Resources.divide(calc, clusterResource, usedResource, clusterResource);
    // assert queue absolute capacity
    assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(), 1e-8);
    // assert user consumed resources.
    assertEquals(usedResource, leafQueue.getUser(app.getUser()).getUsed());
}