Examples with ResourceUsage org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage used on opensource projects

Example 11 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class TestLeafQueue method setUpInternal.

private void setUpInternal(ResourceCalculator rC) throws Exception {
    CapacityScheduler spyCs = new CapacityScheduler();
    queues = new HashMap<String, CSQueue>();
    cs = spy(spyCs);
    rmContext = TestUtils.getMockRMContext();
    spyRMContext = spy(rmContext);
    ConcurrentMap<ApplicationId, RMApp> spyApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>());
    RMApp rmApp = mock(RMApp.class);
    when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any())).thenReturn(null);
    amResourceRequest = mock(ResourceRequest.class);
    when(amResourceRequest.getCapability()).thenReturn(Resources.createResource(0, 0));
    when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
    Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
    when(spyRMContext.getRMApps()).thenReturn(spyApps);
    csConf = new CapacitySchedulerConfiguration();
    csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
    csConf.setBoolean(CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false);
    final String newRoot = "root" + System.currentTimeMillis();
    setupQueueConfiguration(csConf, newRoot);
    YarnConfiguration conf = new YarnConfiguration();
    cs.setConf(conf);
    csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(conf);
    when(csContext.getMinimumResourceCapability()).thenReturn(Resources.createResource(GB, 1));
    when(csContext.getMaximumResourceCapability()).thenReturn(Resources.createResource(16 * GB, 32));
    when(csContext.getClusterResource()).thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(csContext.getResourceCalculator()).thenReturn(rC);
    when(csContext.getRMContext()).thenReturn(rmContext);
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    when(csContext.getContainerTokenSecretManager()).thenReturn(containerTokenSecretManager);
    root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
    ResourceUsage queueResUsage = root.getQueueResourceUsage();
    when(csContext.getClusterResourceUsage()).thenReturn(queueResUsage);
    cs.setRMContext(spyRMContext);
    cs.init(csConf);
    cs.setResourceCalculator(rC);
    cs.start();
    when(spyRMContext.getScheduler()).thenReturn(cs);
    when(spyRMContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
    when(cs.getNumClusterNodes()).thenReturn(3);
}

Example 12 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method mockNested.

ParentQueue mockNested(Resource[] abs, int[] maxCap, Resource[] used, Resource[] pending, Resource[] reserved, int[] apps, Resource[] gran, int[] queues) {
    ResourceCalculator rc = mCS.getResourceCalculator();
    Resource tot = leafAbsCapacities(abs, queues);
    Deque<ParentQueue> pqs = new LinkedList<ParentQueue>();
    ParentQueue root = mockParentQueue(null, queues[0], pqs);
    ResourceUsage resUsage = new ResourceUsage();
    resUsage.setUsed(used[0]);
    resUsage.setReserved(reserved[0]);
    when(root.getQueueName()).thenReturn(CapacitySchedulerConfiguration.ROOT);
    when(root.getAbsoluteUsedCapacity()).thenReturn(Resources.divide(rc, tot, used[0], tot));
    when(root.getAbsoluteCapacity()).thenReturn(Resources.divide(rc, tot, abs[0], tot));
    when(root.getAbsoluteMaximumCapacity()).thenReturn(maxCap[0] / (float) tot.getMemorySize());
    when(root.getQueueResourceUsage()).thenReturn(resUsage);
    QueueCapacities rootQc = new QueueCapacities(true);
    rootQc.setAbsoluteUsedCapacity(Resources.divide(rc, tot, used[0], tot));
    rootQc.setAbsoluteCapacity(Resources.divide(rc, tot, abs[0], tot));
    rootQc.setAbsoluteMaximumCapacity(maxCap[0] / (float) tot.getMemorySize());
    when(root.getQueueCapacities()).thenReturn(rootQc);
    when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT);
    boolean preemptionDisabled = mockPreemptionStatus("root");
    when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
    for (int i = 1; i < queues.length; ++i) {
        final CSQueue q;
        final ParentQueue p = pqs.removeLast();
        final String queueName = "queue" + ((char) ('A' + i - 1));
        if (queues[i] > 0) {
            q = mockParentQueue(p, queues[i], pqs);
            ResourceUsage resUsagePerQueue = new ResourceUsage();
            resUsagePerQueue.setUsed(used[i]);
            resUsagePerQueue.setReserved(reserved[i]);
            when(q.getQueueResourceUsage()).thenReturn(resUsagePerQueue);
        } else {
            q = mockLeafQueue(p, tot, i, abs, used, pending, reserved, apps, gran);
        }
        when(q.getParent()).thenReturn(p);
        when(q.getQueueName()).thenReturn(queueName);
        when(q.getAbsoluteUsedCapacity()).thenReturn(Resources.divide(rc, tot, used[i], tot));
        when(q.getAbsoluteCapacity()).thenReturn(Resources.divide(rc, tot, abs[i], tot));
        when(q.getAbsoluteMaximumCapacity()).thenReturn(maxCap[i] / (float) tot.getMemorySize());
        // We need to make these fields to QueueCapacities
        QueueCapacities qc = new QueueCapacities(false);
        qc.setAbsoluteUsedCapacity(Resources.divide(rc, tot, used[i], tot));
        qc.setAbsoluteCapacity(Resources.divide(rc, tot, abs[i], tot));
        qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize());
        when(q.getQueueCapacities()).thenReturn(qc);
        String parentPathName = p.getQueuePath();
        parentPathName = (parentPathName == null) ? "root" : parentPathName;
        String queuePathName = (parentPathName + "." + queueName).replace("/", "root");
        when(q.getQueuePath()).thenReturn(queuePathName);
        preemptionDisabled = mockPreemptionStatus(queuePathName);
        when(q.getPreemptionDisabled()).thenReturn(preemptionDisabled);
    }
    assert 0 == pqs.size();
    return root;
}

Example 13 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method mockContainers.

private void mockContainers(String containersConfig, FiCaSchedulerApp app, ApplicationAttemptId attemptId, String queueName, List<RMContainer> reservedContainers, List<RMContainer> liveContainers) {
    int containerId = 1;
    int start = containersConfig.indexOf("=") + 1;
    int end = -1;
    Resource used = Resource.newInstance(0, 0);
    Resource pending = Resource.newInstance(0, 0);
    Priority pri = Priority.newInstance(0);
    while (start < containersConfig.length()) {
        while (start < containersConfig.length() && containersConfig.charAt(start) != '(') {
            start++;
        }
        if (start >= containersConfig.length()) {
            throw new IllegalArgumentException("Error containers specification, line=" + containersConfig);
        }
        end = start + 1;
        while (end < containersConfig.length() && containersConfig.charAt(end) != ')') {
            end++;
        }
        if (end >= containersConfig.length()) {
            throw new IllegalArgumentException("Error containers specification, line=" + containersConfig);
        }
        // now we found start/end, get container values
        String[] values = containersConfig.substring(start + 1, end).split(",");
        if (values.length < 6 || values.length > 8) {
            throw new IllegalArgumentException("Format to define container is:" + "(priority,resource,host,expression,repeat,reserved, pending)");
        }
        pri.setPriority(Integer.valueOf(values[0]));
        Resource res = parseResourceFromString(values[1]);
        NodeId host = NodeId.newInstance(values[2], 1);
        String label = values[3];
        String userName = "user";
        int repeat = Integer.valueOf(values[4]);
        boolean reserved = Boolean.valueOf(values[5]);
        if (values.length >= 7) {
            Resources.addTo(pending, parseResourceFromString(values[6]));
        }
        if (values.length == 8) {
            userName = values[7];
        }
        for (int i = 0; i < repeat; i++) {
            Container c = mock(Container.class);
            Resources.addTo(used, res);
            when(c.getResource()).thenReturn(res);
            when(c.getPriority()).thenReturn(pri);
            SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(c);
            RMContainerImpl rmc = mock(RMContainerImpl.class);
            when(rmc.getAllocatedSchedulerKey()).thenReturn(sk);
            when(rmc.getAllocatedNode()).thenReturn(host);
            when(rmc.getNodeLabelExpression()).thenReturn(label);
            when(rmc.getAllocatedResource()).thenReturn(res);
            when(rmc.getContainer()).thenReturn(c);
            when(rmc.getApplicationAttemptId()).thenReturn(attemptId);
            when(rmc.getQueueName()).thenReturn(queueName);
            final ContainerId cId = ContainerId.newContainerId(attemptId, containerId);
            when(rmc.getContainerId()).thenReturn(cId);
            doAnswer(new Answer<Integer>() {

                @Override
                public Integer answer(InvocationOnMock invocation) throws Throwable {
                    return cId.compareTo(((RMContainer) invocation.getArguments()[0]).getContainerId());
                }
            }).when(rmc).compareTo(any(RMContainer.class));
            if (containerId == 1) {
                when(rmc.isAMContainer()).thenReturn(true);
                when(app.getAMResource(label)).thenReturn(res);
            }
            if (reserved) {
                reservedContainers.add(rmc);
                when(rmc.getReservedResource()).thenReturn(res);
            } else {
                liveContainers.add(rmc);
            }
            // Add container to scheduler-node
            addContainerToSchedulerNode(host, rmc, reserved);
            // If this is a non-exclusive allocation
            String partition = null;
            if (label.isEmpty() && !(partition = nodeIdToSchedulerNodes.get(host).getPartition()).isEmpty()) {
                LeafQueue queue = (LeafQueue) nameToCSQueues.get(queueName);
                Map<String, TreeSet<RMContainer>> ignoreExclusivityContainers = queue.getIgnoreExclusivityRMContainers();
                if (!ignoreExclusivityContainers.containsKey(partition)) {
                    ignoreExclusivityContainers.put(partition, new TreeSet<RMContainer>());
                }
                ignoreExclusivityContainers.get(partition).add(rmc);
            }
            LOG.debug("add container to app=" + attemptId + " res=" + res + " node=" + host + " nodeLabelExpression=" + label + " partition=" + partition);
            containerId++;
        }
        // Some more app specific aggregated data can be better filled here.
        when(app.getPriority()).thenReturn(pri);
        when(app.getUser()).thenReturn(userName);
        when(app.getCurrentConsumption()).thenReturn(used);
        when(app.getCurrentReservation()).thenReturn(Resources.createResource(0, 0));
        Map<String, Resource> pendingForDefaultPartition = new HashMap<String, Resource>();
        // Add for default partition for now.
        pendingForDefaultPartition.put(label, pending);
        when(app.getTotalPendingRequestsPerPartition()).thenReturn(pendingForDefaultPartition);
        // need to set pending resource in resource usage as well
        ResourceUsage ru = new ResourceUsage();
        ru.setUsed(label, used);
        when(app.getAppAttemptResourceUsage()).thenReturn(ru);
        start = end + 1;
    }
}

Example 14 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method checkReservedResource.

public void checkReservedResource(CSQueue queue, String partition, int reserved) {
    ResourceUsage ru = queue.getQueueResourceUsage();
    Assert.assertEquals(reserved, ru.getReserved(partition).getMemorySize());
}

Example 15 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method setupQueue.

private void setupQueue(CSQueue queue, String q, String[] queueExprArray, int idx) {
    LOG.debug("*** Setup queue, source=" + q);
    String queuePath = null;
    int myLevel = getLevel(q);
    if (0 == myLevel) {
        // It's root
        when(queue.getQueueName()).thenReturn(ROOT);
        queuePath = ROOT;
    }
    String queueName = getQueueName(q);
    when(queue.getQueueName()).thenReturn(queueName);
    // Setup parent queue, and add myself to parentQueue.children-list
    ParentQueue parentQueue = getParentQueue(queueExprArray, idx, myLevel);
    if (null != parentQueue) {
        when(queue.getParent()).thenReturn(parentQueue);
        parentQueue.getChildQueues().add(queue);
        // Setup my path
        queuePath = parentQueue.getQueuePath() + "." + queueName;
    }
    when(queue.getQueuePath()).thenReturn(queuePath);
    QueueCapacities qc = new QueueCapacities(0 == myLevel);
    ResourceUsage ru = new ResourceUsage();
    when(queue.getQueueCapacities()).thenReturn(qc);
    when(queue.getQueueResourceUsage()).thenReturn(ru);
    LOG.debug("Setup queue, name=" + queue.getQueueName() + " path=" + queue.getQueuePath());
    LOG.debug("Parent=" + (parentQueue == null ? "null" : parentQueue.getQueueName()));
    // Setup other fields like used resource, guaranteed resource, etc.
    String capacitySettingStr = q.substring(q.indexOf("(") + 1, q.indexOf(")"));
    for (String s : capacitySettingStr.split(",")) {
        String partitionName = s.substring(0, s.indexOf("="));
        String[] values = s.substring(s.indexOf("[") + 1, s.indexOf("]")).split(" ");
        // Add a small epsilon to capacities to avoid truncate when doing
        // Resources.multiply
        float epsilon = 1e-6f;
        Resource totResoucePerPartition = partitionToResource.get(partitionName);
        float absGuaranteed = Resources.divide(rc, totResoucePerPartition, parseResourceFromString(values[0].trim()), totResoucePerPartition) + epsilon;
        float absMax = Resources.divide(rc, totResoucePerPartition, parseResourceFromString(values[1].trim()), totResoucePerPartition) + epsilon;
        float absUsed = Resources.divide(rc, totResoucePerPartition, parseResourceFromString(values[2].trim()), totResoucePerPartition) + epsilon;
        float used = Resources.divide(rc, totResoucePerPartition, parseResourceFromString(values[2].trim()), parseResourceFromString(values[0].trim())) + epsilon;
        Resource pending = parseResourceFromString(values[3].trim());
        qc.setAbsoluteCapacity(partitionName, absGuaranteed);
        qc.setAbsoluteMaximumCapacity(partitionName, absMax);
        qc.setAbsoluteUsedCapacity(partitionName, absUsed);
        qc.setUsedCapacity(partitionName, used);
        when(queue.getUsedCapacity()).thenReturn(used);
        ru.setPending(partitionName, pending);
        // Setup reserved resource if it contained by input config
        Resource reserved = Resources.none();
        if (values.length == 5) {
            reserved = parseResourceFromString(values[4].trim());
            ru.setReserved(partitionName, reserved);
        }
        if (!isParent(queueExprArray, idx)) {
            LeafQueue lq = (LeafQueue) queue;
            when(lq.getTotalPendingResourcesConsideringUserLimit(isA(Resource.class), isA(String.class), eq(false))).thenReturn(pending);
            when(lq.getTotalPendingResourcesConsideringUserLimit(isA(Resource.class), isA(String.class), eq(true))).thenReturn(Resources.subtract(pending, reserved));
        }
        ru.setUsed(partitionName, parseResourceFromString(values[2].trim()));
        LOG.debug("Setup queue=" + queueName + " partition=" + partitionName + " [abs_guaranteed=" + absGuaranteed + ",abs_max=" + absMax + ",abs_used" + absUsed + ",pending_resource=" + pending + ", reserved_resource=" + reserved + "]");
    }
    // Setup preemption disabled
    when(queue.getPreemptionDisabled()).thenReturn(conf.getPreemptionDisabled(queuePath, false));
    // Setup other queue configurations
    Map<String, String> otherConfigs = getOtherConfigurations(queueExprArray[idx]);
    if (otherConfigs.containsKey("priority")) {
        when(queue.getPriority()).thenReturn(Priority.newInstance(Integer.valueOf(otherConfigs.get("priority"))));
    } else {
        // set queue's priority to 0 by default
        when(queue.getPriority()).thenReturn(Priority.newInstance(0));
    }
    // Setup disable preemption of queues
    if (otherConfigs.containsKey("disable_preemption")) {
        when(queue.getPreemptionDisabled()).thenReturn(Boolean.valueOf(otherConfigs.get("disable_preemption")));
    }
    nameToCSQueues.put(queueName, queue);
    when(cs.getQueue(eq(queueName))).thenReturn(queue);
}