Examples with SchedulerRequestKey org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey used on opensource projects

Example 26 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class ContainerUpdateContext method checkAndAddToOutstandingDecreases.

/**
   * Add the container to outstanding decreases.
   * @param updateReq UpdateContainerRequest.
   * @param schedulerNode SchedulerNode.
   * @param container Container.
   * @return If it was possible to decrease the container.
   */
public synchronized boolean checkAndAddToOutstandingDecreases(UpdateContainerRequest updateReq, SchedulerNode schedulerNode, Container container) {
    if (outstandingDecreases.containsKey(container.getId())) {
        return false;
    }
    if (ContainerUpdateType.DECREASE_RESOURCE == updateReq.getContainerUpdateType()) {
        SchedulerRequestKey updateKey = new SchedulerRequestKey(container.getPriority(), container.getAllocationRequestId(), container.getId());
        cancelPreviousRequest(schedulerNode, updateKey);
        outstandingDecreases.put(container.getId(), updateReq.getCapability());
    } else {
        outstandingDecreases.put(container.getId(), container.getResource());
    }
    return true;
}

Example 27 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method mockContainer.

RMContainer mockContainer(ApplicationAttemptId appAttId, int id, Resource r, int cpriority) {
    ContainerId cId = ContainerId.newContainerId(appAttId, id);
    Container c = mock(Container.class);
    when(c.getResource()).thenReturn(r);
    when(c.getPriority()).thenReturn(Priority.newInstance(cpriority));
    SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(c);
    RMContainer mC = mock(RMContainer.class);
    when(mC.getContainerId()).thenReturn(cId);
    when(mC.getAllocatedSchedulerKey()).thenReturn(sk);
    when(mC.getContainer()).thenReturn(c);
    when(mC.getApplicationAttemptId()).thenReturn(appAttId);
    when(mC.getAllocatedResource()).thenReturn(r);
    if (priority.AMCONTAINER.getValue() == cpriority) {
        when(mC.isAMContainer()).thenReturn(true);
    }
    if (priority.LABELEDCONTAINER.getValue() == cpriority) {
        when(mC.getAllocatedNode()).thenReturn(NodeId.newInstance("node1", 0));
    }
    return mC;
}

Example 28 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method mockContainers.

private void mockContainers(String containersConfig, FiCaSchedulerApp app, ApplicationAttemptId attemptId, String queueName, List<RMContainer> reservedContainers, List<RMContainer> liveContainers) {
    int containerId = 1;
    int start = containersConfig.indexOf("=") + 1;
    int end = -1;
    Resource used = Resource.newInstance(0, 0);
    Resource pending = Resource.newInstance(0, 0);
    Priority pri = Priority.newInstance(0);
    while (start < containersConfig.length()) {
        while (start < containersConfig.length() && containersConfig.charAt(start) != '(') {
            start++;
        }
        if (start >= containersConfig.length()) {
            throw new IllegalArgumentException("Error containers specification, line=" + containersConfig);
        }
        end = start + 1;
        while (end < containersConfig.length() && containersConfig.charAt(end) != ')') {
            end++;
        }
        if (end >= containersConfig.length()) {
            throw new IllegalArgumentException("Error containers specification, line=" + containersConfig);
        }
        // now we found start/end, get container values
        String[] values = containersConfig.substring(start + 1, end).split(",");
        if (values.length < 6 || values.length > 8) {
            throw new IllegalArgumentException("Format to define container is:" + "(priority,resource,host,expression,repeat,reserved, pending)");
        }
        pri.setPriority(Integer.valueOf(values[0]));
        Resource res = parseResourceFromString(values[1]);
        NodeId host = NodeId.newInstance(values[2], 1);
        String label = values[3];
        String userName = "user";
        int repeat = Integer.valueOf(values[4]);
        boolean reserved = Boolean.valueOf(values[5]);
        if (values.length >= 7) {
            Resources.addTo(pending, parseResourceFromString(values[6]));
        }
        if (values.length == 8) {
            userName = values[7];
        }
        for (int i = 0; i < repeat; i++) {
            Container c = mock(Container.class);
            Resources.addTo(used, res);
            when(c.getResource()).thenReturn(res);
            when(c.getPriority()).thenReturn(pri);
            SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(c);
            RMContainerImpl rmc = mock(RMContainerImpl.class);
            when(rmc.getAllocatedSchedulerKey()).thenReturn(sk);
            when(rmc.getAllocatedNode()).thenReturn(host);
            when(rmc.getNodeLabelExpression()).thenReturn(label);
            when(rmc.getAllocatedResource()).thenReturn(res);
            when(rmc.getContainer()).thenReturn(c);
            when(rmc.getApplicationAttemptId()).thenReturn(attemptId);
            when(rmc.getQueueName()).thenReturn(queueName);
            final ContainerId cId = ContainerId.newContainerId(attemptId, containerId);
            when(rmc.getContainerId()).thenReturn(cId);
            doAnswer(new Answer<Integer>() {

                @Override
                public Integer answer(InvocationOnMock invocation) throws Throwable {
                    return cId.compareTo(((RMContainer) invocation.getArguments()[0]).getContainerId());
                }
            }).when(rmc).compareTo(any(RMContainer.class));
            if (containerId == 1) {
                when(rmc.isAMContainer()).thenReturn(true);
                when(app.getAMResource(label)).thenReturn(res);
            }
            if (reserved) {
                reservedContainers.add(rmc);
                when(rmc.getReservedResource()).thenReturn(res);
            } else {
                liveContainers.add(rmc);
            }
            // Add container to scheduler-node
            addContainerToSchedulerNode(host, rmc, reserved);
            // If this is a non-exclusive allocation
            String partition = null;
            if (label.isEmpty() && !(partition = nodeIdToSchedulerNodes.get(host).getPartition()).isEmpty()) {
                LeafQueue queue = (LeafQueue) nameToCSQueues.get(queueName);
                Map<String, TreeSet<RMContainer>> ignoreExclusivityContainers = queue.getIgnoreExclusivityRMContainers();
                if (!ignoreExclusivityContainers.containsKey(partition)) {
                    ignoreExclusivityContainers.put(partition, new TreeSet<RMContainer>());
                }
                ignoreExclusivityContainers.get(partition).add(rmc);
            }
            LOG.debug("add container to app=" + attemptId + " res=" + res + " node=" + host + " nodeLabelExpression=" + label + " partition=" + partition);
            containerId++;
        }
        // Some more app specific aggregated data can be better filled here.
        when(app.getPriority()).thenReturn(pri);
        when(app.getUser()).thenReturn(userName);
        when(app.getCurrentConsumption()).thenReturn(used);
        when(app.getCurrentReservation()).thenReturn(Resources.createResource(0, 0));
        Map<String, Resource> pendingForDefaultPartition = new HashMap<String, Resource>();
        // Add for default partition for now.
        pendingForDefaultPartition.put(label, pending);
        when(app.getTotalPendingRequestsPerPartition()).thenReturn(pendingForDefaultPartition);
        // need to set pending resource in resource usage as well
        ResourceUsage ru = new ResourceUsage();
        ru.setUsed(label, used);
        when(app.getAppAttemptResourceUsage()).thenReturn(ru);
        start = end + 1;
    }
}

Example 29 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class TestAbstractYarnScheduler method testResourceRequestRecoveryToTheRightAppAttempt.

/**
   * Test to verify that ResourceRequests recovery back to the right app-attempt
   * after a container gets killed at ACQUIRED state: YARN-4502.
   *
   * @throws Exception
   */
@Test
public void testResourceRequestRecoveryToTheRightAppAttempt() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        RMApp rmApp = rm.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", -1, null, "Test", false, true);
        MockNM node = new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
        node.registerNode();
        MockAM am1 = MockRM.launchAndRegisterAM(rmApp, rm, node);
        ApplicationAttemptId applicationAttemptOneID = am1.getApplicationAttemptId();
        ContainerId am1ContainerID = ContainerId.newContainerId(applicationAttemptOneID, 1);
        // allocate NUM_CONTAINERS containers
        am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
        node.nodeHeartbeat(true);
        // wait for containers to be allocated.
        List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        while (containers.size() != 1) {
            node.nodeHeartbeat(true);
            containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
            Thread.sleep(200);
        }
        // launch a 2nd container, for testing running-containers transfer.
        node.nodeHeartbeat(applicationAttemptOneID, 2, ContainerState.RUNNING);
        ContainerId runningContainerID = ContainerId.newContainerId(applicationAttemptOneID, 2);
        rm.waitForState(node, runningContainerID, RMContainerState.RUNNING);
        // 3rd container is in Allocated state.
        int ALLOCATED_CONTAINER_PRIORITY = 1047;
        am1.allocate("127.0.0.1", 1024, 1, ALLOCATED_CONTAINER_PRIORITY, new ArrayList<ContainerId>(), null);
        node.nodeHeartbeat(true);
        ContainerId allocatedContainerID = ContainerId.newContainerId(applicationAttemptOneID, 3);
        rm.waitForState(node, allocatedContainerID, RMContainerState.ALLOCATED);
        RMContainer allocatedContainer = rm.getResourceScheduler().getRMContainer(allocatedContainerID);
        // Capture scheduler app-attempt before AM crash.
        SchedulerApplicationAttempt firstSchedulerAppAttempt = ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm.getResourceScheduler()).getApplicationAttempt(applicationAttemptOneID);
        // AM crashes, and a new app-attempt gets created
        node.nodeHeartbeat(applicationAttemptOneID, 1, ContainerState.COMPLETE);
        rm.drainEvents();
        RMAppAttempt rmAppAttempt2 = MockRM.waitForAttemptScheduled(rmApp, rm);
        ApplicationAttemptId applicationAttemptTwoID = rmAppAttempt2.getAppAttemptId();
        Assert.assertEquals(2, applicationAttemptTwoID.getAttemptId());
        // All outstanding allocated containers will be killed (irrespective of
        // keep-alive of container across app-attempts)
        Assert.assertEquals(RMContainerState.KILLED, allocatedContainer.getState());
        // original app-attempt, not the new one
        for (SchedulerRequestKey key : firstSchedulerAppAttempt.getSchedulerKeys()) {
            if (key.getPriority().getPriority() == 0) {
                Assert.assertEquals(0, firstSchedulerAppAttempt.getOutstandingAsksCount(key));
            } else if (key.getPriority().getPriority() == ALLOCATED_CONTAINER_PRIORITY) {
                Assert.assertEquals(1, firstSchedulerAppAttempt.getOutstandingAsksCount(key));
            }
        }
        // Also, only one running container should be transferred after AM
        // launches
        MockRM.launchAM(rmApp, rm, node);
        List<Container> transferredContainers = rm.getResourceScheduler().getTransferredContainers(applicationAttemptTwoID);
        Assert.assertEquals(1, transferredContainers.size());
        Assert.assertEquals(runningContainerID, transferredContainers.get(0).getId());
    } finally {
        rm.stop();
    }
}

Example 30 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class TestAppSchedulingInfo method testSchedulerRequestKeyOrdering.

@Test
public void testSchedulerRequestKeyOrdering() {
    TreeSet<SchedulerRequestKey> ts = new TreeSet<>();
    ts.add(TestUtils.toSchedulerKey(Priority.newInstance(1), 1));
    ts.add(TestUtils.toSchedulerKey(Priority.newInstance(1), 2));
    ts.add(TestUtils.toSchedulerKey(Priority.newInstance(0), 4));
    ts.add(TestUtils.toSchedulerKey(Priority.newInstance(0), 3));
    ts.add(TestUtils.toSchedulerKey(Priority.newInstance(2), 5));
    ts.add(TestUtils.toSchedulerKey(Priority.newInstance(2), 6));
    Iterator<SchedulerRequestKey> iter = ts.iterator();
    SchedulerRequestKey sk = iter.next();
    Assert.assertEquals(0, sk.getPriority().getPriority());
    Assert.assertEquals(3, sk.getAllocationRequestId());
    sk = iter.next();
    Assert.assertEquals(0, sk.getPriority().getPriority());
    Assert.assertEquals(4, sk.getAllocationRequestId());
    sk = iter.next();
    Assert.assertEquals(1, sk.getPriority().getPriority());
    Assert.assertEquals(1, sk.getAllocationRequestId());
    sk = iter.next();
    Assert.assertEquals(1, sk.getPriority().getPriority());
    Assert.assertEquals(2, sk.getAllocationRequestId());
    sk = iter.next();
    Assert.assertEquals(2, sk.getPriority().getPriority());
    Assert.assertEquals(5, sk.getAllocationRequestId());
    sk = iter.next();
    Assert.assertEquals(2, sk.getPriority().getPriority());
    Assert.assertEquals(6, sk.getAllocationRequestId());
}