Examples with RMContainer org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer used on opensource projects

Example 46 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class LeafQueue method allocateResource.

void allocateResource(Resource clusterResource, SchedulerApplicationAttempt application, Resource resource, String nodePartition, RMContainer rmContainer) {
    try {
        writeLock.lock();
        super.allocateResource(clusterResource, resource, nodePartition);
        // handle ignore exclusivity container
        if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
            TreeSet<RMContainer> rmContainers = null;
            if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get(nodePartition))) {
                rmContainers = new TreeSet<>();
                ignorePartitionExclusivityRMContainers.put(nodePartition, rmContainers);
            }
            rmContainers.add(rmContainer);
        }
        // Update user metrics
        String userName = application.getUser();
        // Increment user's resource usage.
        User user = usersManager.updateUserResourceUsage(userName, resource, nodePartition, true);
        // Note this is a bit unconventional since it gets the object and modifies
        // it here, rather then using set routine
        // headroom
        Resources.subtractFrom(application.getHeadroom(), resource);
        metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
        if (LOG.isDebugEnabled()) {
            LOG.debug(getQueueName() + " user=" + userName + " used=" + queueUsage.getUsed(nodePartition) + " numContainers=" + numContainers + " headroom = " + application.getHeadroom() + " user-resources=" + user.getUsed());
        }
    } finally {
        writeLock.unlock();
    }
}

Example 47 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class LeafQueue method releaseResource.

void releaseResource(Resource clusterResource, FiCaSchedulerApp application, Resource resource, String nodePartition, RMContainer rmContainer) {
    try {
        writeLock.lock();
        super.releaseResource(clusterResource, resource, nodePartition);
        // handle ignore exclusivity container
        if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
            if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) {
                Set<RMContainer> rmContainers = ignorePartitionExclusivityRMContainers.get(nodePartition);
                rmContainers.remove(rmContainer);
                if (rmContainers.isEmpty()) {
                    ignorePartitionExclusivityRMContainers.remove(nodePartition);
                }
            }
        }
        // Update user metrics
        String userName = application.getUser();
        User user = usersManager.updateUserResourceUsage(userName, resource, nodePartition, false);
        metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
        if (LOG.isDebugEnabled()) {
            LOG.debug(getQueueName() + " used=" + queueUsage.getUsed() + " numContainers=" + numContainers + " user=" + userName + " user-resources=" + user.getUsed());
        }
    } finally {
        writeLock.unlock();
    }
}

Example 48 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class LeafQueue method internalReleaseContainer.

private void internalReleaseContainer(Resource clusterResource, SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer) {
    RMContainer rmContainer = schedulerContainer.getRmContainer();
    LeafQueue targetLeafQueue = schedulerContainer.getSchedulerApplicationAttempt().getCSLeafQueue();
    if (targetLeafQueue == this) {
        // When trying to preempt containers from the same queue
        if (rmContainer.getState() == RMContainerState.RESERVED) {
            // For other reserved containers
            // This is a reservation exchange, complete previous reserved container
            completedContainer(clusterResource, schedulerContainer.getSchedulerApplicationAttempt(), schedulerContainer.getSchedulerNode(), rmContainer, SchedulerUtils.createAbnormalContainerStatus(rmContainer.getContainerId(), SchedulerUtils.UNRESERVED_CONTAINER), RMContainerEventType.RELEASED, null, false);
        }
    } else {
        // When trying to preempt containers from different queue -- this
        // is for lazy preemption feature (kill preemption candidate in scheduling
        // cycle).
        targetLeafQueue.completedContainer(clusterResource, schedulerContainer.getSchedulerApplicationAttempt(), schedulerContainer.getSchedulerNode(), schedulerContainer.getRmContainer(), SchedulerUtils.createPreemptedContainerStatus(rmContainer.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL, null, false);
    }
}

Example 49 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class CapacityScheduler method updateLabelsOnNode.

/**
   * Process node labels update on a node.
   */
private void updateLabelsOnNode(NodeId nodeId, Set<String> newLabels) {
    FiCaSchedulerNode node = nodeTracker.getNode(nodeId);
    if (null == node) {
        return;
    }
    // Get new partition, we have only one partition per node
    String newPartition;
    if (newLabels.isEmpty()) {
        newPartition = RMNodeLabelsManager.NO_LABEL;
    } else {
        newPartition = newLabels.iterator().next();
    }
    // old partition as well
    String oldPartition = node.getPartition();
    // Update resources of these containers
    for (RMContainer rmContainer : node.getCopiedListOfRunningContainers()) {
        FiCaSchedulerApp application = getApplicationAttempt(rmContainer.getApplicationAttemptId());
        if (null != application) {
            application.nodePartitionUpdated(rmContainer, oldPartition, newPartition);
        } else {
            LOG.warn("There's something wrong, some RMContainers running on" + " a node, but we cannot find SchedulerApplicationAttempt " + "for it. Node=" + node.getNodeID() + " applicationAttemptId=" + rmContainer.getApplicationAttemptId());
            continue;
        }
    }
    // Unreserve container on this node
    RMContainer reservedContainer = node.getReservedContainer();
    if (null != reservedContainer) {
        killReservedContainer(reservedContainer);
    }
    // Update node labels after we've done this
    node.updateLabels(newLabels);
}

Example 50 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class TestRMAppAttemptTransitions method allocateApplicationAttempt.

@SuppressWarnings("unchecked")
private Container allocateApplicationAttempt() {
    scheduleApplicationAttempt();
    // Mock the allocation of AM container 
    Container container = mock(Container.class);
    Resource resource = BuilderUtils.newResource(2048, 1);
    when(container.getId()).thenReturn(BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 1));
    when(container.getResource()).thenReturn(resource);
    Allocation allocation = mock(Allocation.class);
    when(allocation.getContainers()).thenReturn(Collections.singletonList(container));
    when(scheduler.allocate(any(ApplicationAttemptId.class), any(List.class), any(List.class), any(List.class), any(List.class), any(ContainerUpdates.class))).thenReturn(allocation);
    RMContainer rmContainer = mock(RMContainerImpl.class);
    when(scheduler.getRMContainer(container.getId())).thenReturn(rmContainer);
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.CONTAINER_ALLOCATED));
    assertEquals(RMAppAttemptState.ALLOCATED_SAVING, applicationAttempt.getAppAttemptState());
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
    testAppAttemptAllocatedState(container);
    return container;
}