Examples with RMContainerImpl org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl used on opensource projects

Example 16 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class TestCapacityScheduler method testRemoveAttemptMoveAdded.

@Test
public void testRemoveAttemptMoveAdded() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, CapacityScheduler.class);
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
    // Create Mock RM
    MockRM rm = new MockRM(getCapacityConfiguration(conf));
    CapacityScheduler sch = (CapacityScheduler) rm.getResourceScheduler();
    // add node
    Resource newResource = Resource.newInstance(4 * GB, 1);
    RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1");
    SchedulerEvent addNode = new NodeAddedSchedulerEvent(node);
    sch.handle(addNode);
    // create appid
    ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    RMAppAttemptMetrics attemptMetric = new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
    RMAppImpl app = mock(RMAppImpl.class);
    when(app.getApplicationId()).thenReturn(appId);
    RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
    Container container = mock(Container.class);
    when(attempt.getMasterContainer()).thenReturn(container);
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(attempt.getSubmissionContext()).thenReturn(submissionContext);
    when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
    when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
    when(app.getCurrentAppAttempt()).thenReturn(attempt);
    rm.getRMContext().getRMApps().put(appId, app);
    // Add application
    SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, "a1", "user");
    sch.handle(addAppEvent);
    // Add application attempt
    SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    sch.handle(addAttemptEvent);
    // get Queues
    CSQueue queueA1 = sch.getQueue("a1");
    CSQueue queueB = sch.getQueue("b");
    CSQueue queueB1 = sch.getQueue("b1");
    // add Running rm container and simulate live containers to a1
    ContainerId newContainerId = ContainerId.newContainerId(appAttemptId, 2);
    RMContainerImpl rmContainer = mock(RMContainerImpl.class);
    when(rmContainer.getState()).thenReturn(RMContainerState.RUNNING);
    Container container2 = mock(Container.class);
    when(rmContainer.getContainer()).thenReturn(container2);
    Resource resource = Resource.newInstance(1024, 1);
    when(container2.getResource()).thenReturn(resource);
    when(rmContainer.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
    when(container2.getNodeId()).thenReturn(node.getNodeID());
    when(container2.getId()).thenReturn(newContainerId);
    when(rmContainer.getNodeLabelExpression()).thenReturn(RMNodeLabelsManager.NO_LABEL);
    when(rmContainer.getContainerId()).thenReturn(newContainerId);
    sch.getApplicationAttempt(appAttemptId).getLiveContainersMap().put(newContainerId, rmContainer);
    QueueMetrics queueA1M = queueA1.getMetrics();
    queueA1M.incrPendingResources("user1", 1, resource);
    queueA1M.allocateResources("user1", resource);
    // remove attempt
    sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId, RMAppAttemptState.KILLED, true));
    // Move application to queue b1
    sch.moveApplication(appId, "b1");
    // Check queue metrics after move
    Assert.assertEquals(0, queueA1.getNumApplications());
    Assert.assertEquals(1, queueB.getNumApplications());
    Assert.assertEquals(0, queueB1.getNumApplications());
    // Release attempt add event
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(appId, 2);
    SchedulerEvent addAttemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, true);
    sch.handle(addAttemptEvent2);
    // Check metrics after attempt added
    Assert.assertEquals(0, queueA1.getNumApplications());
    Assert.assertEquals(1, queueB.getNumApplications());
    Assert.assertEquals(1, queueB1.getNumApplications());
    QueueMetrics queueB1M = queueB1.getMetrics();
    QueueMetrics queueBM = queueB.getMetrics();
    // Verify allocation MB of current state
    Assert.assertEquals(0, queueA1M.getAllocatedMB());
    Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores());
    Assert.assertEquals(1024, queueB1M.getAllocatedMB());
    Assert.assertEquals(1, queueB1M.getAllocatedVirtualCores());
    // remove attempt
    sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId2, RMAppAttemptState.FINISHED, false));
    Assert.assertEquals(0, queueA1M.getAllocatedMB());
    Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores());
    Assert.assertEquals(0, queueB1M.getAllocatedMB());
    Assert.assertEquals(0, queueB1M.getAllocatedVirtualCores());
    verifyQueueMetrics(queueB1M);
    verifyQueueMetrics(queueBM);
    // Verify queue A1 metrics
    verifyQueueMetrics(queueA1M);
    rm.close();
}

Example 17 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class FiCaSchedulerApp method allocate.

public RMContainer allocate(FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Container container) {
    try {
        readLock.lock();
        if (isStopped) {
            return null;
        }
        // request without locking the scheduler, hence we need to check
        if (getOutstandingAsksCount(schedulerKey) <= 0) {
            return null;
        }
        SchedulingPlacementSet<FiCaSchedulerNode> ps = appSchedulingInfo.getSchedulingPlacementSet(schedulerKey);
        if (null == ps) {
            LOG.warn("Failed to get " + SchedulingPlacementSet.class.getName() + " for application=" + getApplicationId() + " schedulerRequestKey=" + schedulerKey);
            return null;
        }
        // Create RMContainer
        RMContainer rmContainer = new RMContainerImpl(container, schedulerKey, this.getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), this.rmContext, ps.getPrimaryRequestedNodePartition());
        ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
        // FIXME, should set when confirmed
        updateAMContainerDiagnostics(AMState.ASSIGNED, null);
        return rmContainer;
    } finally {
        readLock.unlock();
    }
}

Example 18 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method mockContainers.

private void mockContainers(String containersConfig, FiCaSchedulerApp app, ApplicationAttemptId attemptId, String queueName, List<RMContainer> reservedContainers, List<RMContainer> liveContainers) {
    int containerId = 1;
    int start = containersConfig.indexOf("=") + 1;
    int end = -1;
    Resource used = Resource.newInstance(0, 0);
    Resource pending = Resource.newInstance(0, 0);
    Priority pri = Priority.newInstance(0);
    while (start < containersConfig.length()) {
        while (start < containersConfig.length() && containersConfig.charAt(start) != '(') {
            start++;
        }
        if (start >= containersConfig.length()) {
            throw new IllegalArgumentException("Error containers specification, line=" + containersConfig);
        }
        end = start + 1;
        while (end < containersConfig.length() && containersConfig.charAt(end) != ')') {
            end++;
        }
        if (end >= containersConfig.length()) {
            throw new IllegalArgumentException("Error containers specification, line=" + containersConfig);
        }
        // now we found start/end, get container values
        String[] values = containersConfig.substring(start + 1, end).split(",");
        if (values.length < 6 || values.length > 8) {
            throw new IllegalArgumentException("Format to define container is:" + "(priority,resource,host,expression,repeat,reserved, pending)");
        }
        pri.setPriority(Integer.valueOf(values[0]));
        Resource res = parseResourceFromString(values[1]);
        NodeId host = NodeId.newInstance(values[2], 1);
        String label = values[3];
        String userName = "user";
        int repeat = Integer.valueOf(values[4]);
        boolean reserved = Boolean.valueOf(values[5]);
        if (values.length >= 7) {
            Resources.addTo(pending, parseResourceFromString(values[6]));
        }
        if (values.length == 8) {
            userName = values[7];
        }
        for (int i = 0; i < repeat; i++) {
            Container c = mock(Container.class);
            Resources.addTo(used, res);
            when(c.getResource()).thenReturn(res);
            when(c.getPriority()).thenReturn(pri);
            SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(c);
            RMContainerImpl rmc = mock(RMContainerImpl.class);
            when(rmc.getAllocatedSchedulerKey()).thenReturn(sk);
            when(rmc.getAllocatedNode()).thenReturn(host);
            when(rmc.getNodeLabelExpression()).thenReturn(label);
            when(rmc.getAllocatedResource()).thenReturn(res);
            when(rmc.getContainer()).thenReturn(c);
            when(rmc.getApplicationAttemptId()).thenReturn(attemptId);
            when(rmc.getQueueName()).thenReturn(queueName);
            final ContainerId cId = ContainerId.newContainerId(attemptId, containerId);
            when(rmc.getContainerId()).thenReturn(cId);
            doAnswer(new Answer<Integer>() {

                @Override
                public Integer answer(InvocationOnMock invocation) throws Throwable {
                    return cId.compareTo(((RMContainer) invocation.getArguments()[0]).getContainerId());
                }
            }).when(rmc).compareTo(any(RMContainer.class));
            if (containerId == 1) {
                when(rmc.isAMContainer()).thenReturn(true);
                when(app.getAMResource(label)).thenReturn(res);
            }
            if (reserved) {
                reservedContainers.add(rmc);
                when(rmc.getReservedResource()).thenReturn(res);
            } else {
                liveContainers.add(rmc);
            }
            // Add container to scheduler-node
            addContainerToSchedulerNode(host, rmc, reserved);
            // If this is a non-exclusive allocation
            String partition = null;
            if (label.isEmpty() && !(partition = nodeIdToSchedulerNodes.get(host).getPartition()).isEmpty()) {
                LeafQueue queue = (LeafQueue) nameToCSQueues.get(queueName);
                Map<String, TreeSet<RMContainer>> ignoreExclusivityContainers = queue.getIgnoreExclusivityRMContainers();
                if (!ignoreExclusivityContainers.containsKey(partition)) {
                    ignoreExclusivityContainers.put(partition, new TreeSet<RMContainer>());
                }
                ignoreExclusivityContainers.get(partition).add(rmc);
            }
            LOG.debug("add container to app=" + attemptId + " res=" + res + " node=" + host + " nodeLabelExpression=" + label + " partition=" + partition);
            containerId++;
        }
        // Some more app specific aggregated data can be better filled here.
        when(app.getPriority()).thenReturn(pri);
        when(app.getUser()).thenReturn(userName);
        when(app.getCurrentConsumption()).thenReturn(used);
        when(app.getCurrentReservation()).thenReturn(Resources.createResource(0, 0));
        Map<String, Resource> pendingForDefaultPartition = new HashMap<String, Resource>();
        // Add for default partition for now.
        pendingForDefaultPartition.put(label, pending);
        when(app.getTotalPendingRequestsPerPartition()).thenReturn(pendingForDefaultPartition);
        // need to set pending resource in resource usage as well
        ResourceUsage ru = new ResourceUsage();
        ru.setUsed(label, used);
        when(app.getAppAttemptResourceUsage()).thenReturn(ru);
        start = end + 1;
    }
}

Example 19 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class SchedulerUtils method createOpportunisticRmContainer.

public static RMContainer createOpportunisticRmContainer(RMContext rmContext, Container container, boolean isRemotelyAllocated) {
    SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) rmContext.getScheduler()).getCurrentAttemptForContainer(container.getId());
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttempt.getApplicationAttemptId(), container.getNodeId(), appAttempt.getUser(), rmContext, isRemotelyAllocated);
    appAttempt.addRMContainer(container.getId(), rmContainer);
    ((AbstractYarnScheduler) rmContext.getScheduler()).getNode(container.getNodeId()).allocateContainer(rmContainer);
    return rmContainer;
}