Examples with RMContainer org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer used on opensource projects

Example 81 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class AbstractYarnScheduler method containerIncreasedOnNode.

protected void containerIncreasedOnNode(ContainerId containerId, SchedulerNode node, Container increasedContainerReportedByNM) {
    /*
     * No lock is required, as this method is protected by scheduler's writeLock
     */
    // Get the application for the finished container
    SchedulerApplicationAttempt application = getCurrentAttemptForContainer(containerId);
    if (application == null) {
        LOG.info("Unknown application " + containerId.getApplicationAttemptId().getApplicationId() + " increased container " + containerId + " on node: " + node);
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId));
        return;
    }
    RMContainer rmContainer = getRMContainer(containerId);
    if (rmContainer == null) {
        // Some unknown container sneaked into the system. Kill it.
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId));
        return;
    }
    rmContainer.handle(new RMContainerNMDoneChangeResourceEvent(containerId, increasedContainerReportedByNM.getResource()));
}

Example 82 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class AbstractYarnScheduler method updateCompletedContainers.

/**
   * Process completed container list.
   * @param completedContainers Extracted list of completed containers
   * @param releasedResources Reference resource object for completed containers
   * @param nodeId NodeId corresponding to the NodeManager
   * @return The total number of released containers
   */
protected int updateCompletedContainers(List<ContainerStatus> completedContainers, Resource releasedResources, NodeId nodeId) {
    int releasedContainers = 0;
    SchedulerNode node = getNode(nodeId);
    List<ContainerId> untrackedContainerIdList = new ArrayList<ContainerId>();
    for (ContainerStatus completedContainer : completedContainers) {
        ContainerId containerId = completedContainer.getContainerId();
        LOG.debug("Container FINISHED: " + containerId);
        RMContainer container = getRMContainer(containerId);
        completedContainer(container, completedContainer, RMContainerEventType.FINISHED);
        if (node != null) {
            node.releaseContainer(containerId, true);
        }
        if (container != null) {
            releasedContainers++;
            Resource ars = container.getAllocatedResource();
            if (ars != null) {
                Resources.addTo(releasedResources, ars);
            }
            Resource rrs = container.getReservedResource();
            if (rrs != null) {
                Resources.addTo(releasedResources, rrs);
            }
        } else {
            // Add containers which are untracked by RM.
            untrackedContainerIdList.add(containerId);
        }
    }
    // Acknowledge NM to remove RM-untracked-containers from NM context.
    if (!untrackedContainerIdList.isEmpty()) {
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId, untrackedContainerIdList));
    }
    return releasedContainers;
}

Example 83 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class TestSystemMetricsPublisherForV2 method testPublishContainerMetrics.

@Test(timeout = 10000)
public void testPublishContainerMetrics() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    RMApp app = rmAppsMapInContext.get(appId);
    if (app == null) {
        app = createAppAndRegister(appId);
    }
    ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId, 1), 1);
    RMContainer container = createRMContainer(containerId);
    metricsPublisher.containerCreated(container, container.getCreationTime());
    metricsPublisher.containerFinished(container, container.getFinishTime());
    dispatcher.await();
    String outputDirApp = getTimelineEntityDir(app) + "/" + TimelineEntityType.YARN_CONTAINER + "/";
    File entityFolder = new File(outputDirApp);
    Assert.assertTrue(entityFolder.isDirectory());
    // file name is <entityId>.thist
    String timelineServiceFileName = containerId.toString() + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
    File appFile = new File(outputDirApp, timelineServiceFileName);
    Assert.assertTrue(appFile.exists());
    verifyEntity(appFile, 2, ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0);
}

Example 84 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method mockApplications.

/**
   * Format is:
   * <pre>
   * queueName\t  // app1
   * (priority,resource,host,expression,#repeat,reserved)
   * (priority,resource,host,expression,#repeat,reserved);
   * queueName\t  // app2
   * </pre>
   */
private void mockApplications(String appsConfig) {
    int id = 1;
    HashMap<String, HashSet<String>> userMap = new HashMap<String, HashSet<String>>();
    LeafQueue queue = null;
    for (String a : appsConfig.split(";")) {
        String[] strs = a.split("\t");
        String queueName = strs[0];
        // get containers
        List<RMContainer> liveContainers = new ArrayList<RMContainer>();
        List<RMContainer> reservedContainers = new ArrayList<RMContainer>();
        ApplicationId appId = ApplicationId.newInstance(0L, id);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        FiCaSchedulerApp app = mock(FiCaSchedulerApp.class);
        when(app.getAMResource(anyString())).thenReturn(Resources.createResource(0, 0));
        mockContainers(strs[1], app, appAttemptId, queueName, reservedContainers, liveContainers);
        LOG.debug("Application mock: queue: " + queueName + ", appId:" + appId);
        when(app.getLiveContainers()).thenReturn(liveContainers);
        when(app.getReservedContainers()).thenReturn(reservedContainers);
        when(app.getApplicationAttemptId()).thenReturn(appAttemptId);
        when(app.getApplicationId()).thenReturn(appId);
        // add to LeafQueue
        queue = (LeafQueue) nameToCSQueues.get(queueName);
        queue.getApplications().add(app);
        queue.getAllApplications().add(app);
        HashSet<String> users = userMap.get(queueName);
        if (null == users) {
            users = new HashSet<String>();
            userMap.put(queueName, users);
        }
        users.add(app.getUser());
        id++;
    }
    for (String queueName : userMap.keySet()) {
        queue = (LeafQueue) nameToCSQueues.get(queueName);
        // Currently we have user-limit test support only for default label.
        Resource totResoucePerPartition = partitionToResource.get("");
        Resource capacity = Resources.multiply(totResoucePerPartition, queue.getQueueCapacities().getAbsoluteCapacity());
        HashSet<String> users = userMap.get(queue.getQueueName());
        Resource userLimit = Resources.divideAndCeil(rc, capacity, users.size());
        for (String userName : users) {
            when(queue.getResourceLimitForAllUsers(eq(userName), any(Resource.class), anyString(), any(SchedulingMode.class))).thenReturn(userLimit);
        }
    }
}

Example 85 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method checkContainerNodesInApp.

public void checkContainerNodesInApp(FiCaSchedulerApp app, int expectedContainersNumber, String host) {
    NodeId nodeId = NodeId.newInstance(host, 1);
    int num = 0;
    for (RMContainer c : app.getLiveContainers()) {
        if (c.getAllocatedNode().equals(nodeId)) {
            num++;
        }
    }
    for (RMContainer c : app.getReservedContainers()) {
        if (c.getAllocatedNode().equals(nodeId)) {
            num++;
        }
    }
    Assert.assertEquals(expectedContainersNumber, num);
}