Examples with Container org.apache.hadoop.yarn.api.records.Container used on opensource projects

Example 31 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class ContainerUpdateContext method checkAndAddToOutstandingIncreases.

/**
   * Add the container to outstanding increases.
   * @param rmContainer RMContainer.
   * @param schedulerNode SchedulerNode.
   * @param updateRequest UpdateContainerRequest.
   * @return true if updated to outstanding increases was successful.
   */
public synchronized boolean checkAndAddToOutstandingIncreases(RMContainer rmContainer, SchedulerNode schedulerNode, UpdateContainerRequest updateRequest) {
    Container container = rmContainer.getContainer();
    SchedulerRequestKey schedulerKey = SchedulerRequestKey.create(updateRequest, rmContainer.getAllocatedSchedulerKey());
    Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap = outstandingIncreases.get(schedulerKey);
    if (resourceMap == null) {
        resourceMap = new HashMap<>();
        outstandingIncreases.put(schedulerKey, resourceMap);
    } else {
        // Updating Resource for and existing increase container
        if (ContainerUpdateType.INCREASE_RESOURCE == updateRequest.getContainerUpdateType()) {
            cancelPreviousRequest(schedulerNode, schedulerKey);
        } else {
            return false;
        }
    }
    Resource resToIncrease = getResourceToIncrease(updateRequest, rmContainer);
    Map<NodeId, Set<ContainerId>> locationMap = resourceMap.get(resToIncrease);
    if (locationMap == null) {
        locationMap = new HashMap<>();
        resourceMap.put(resToIncrease, locationMap);
    }
    Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
    if (containerIds == null) {
        containerIds = new HashSet<>();
        locationMap.put(container.getNodeId(), containerIds);
    }
    if (outstandingDecreases.containsKey(container.getId())) {
        return false;
    }
    containerIds.add(container.getId());
    if (!Resources.isNone(resToIncrease)) {
        Map<SchedulerRequestKey, Map<String, ResourceRequest>> updateResReqs = new HashMap<>();
        Map<String, ResourceRequest> resMap = createResourceRequests(rmContainer, schedulerNode, schedulerKey, resToIncrease);
        updateResReqs.put(schedulerKey, resMap);
        appSchedulingInfo.addToPlacementSets(false, updateResReqs);
    }
    return true;
}

Example 32 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class LeafQueue method completedContainer.

@Override
public void completedContainer(Resource clusterResource, FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue, boolean sortQueues) {
    // Update SchedulerHealth for released / preempted container
    updateSchedulerHealthForCompletedContainer(rmContainer, containerStatus);
    if (application != null) {
        boolean removed = false;
        // Careful! Locking order is important!
        try {
            writeLock.lock();
            Container container = rmContainer.getContainer();
            // So, this is, in effect, a transaction across application & node
            if (rmContainer.getState() == RMContainerState.RESERVED) {
                removed = application.unreserve(rmContainer.getReservedSchedulerKey(), node, rmContainer);
            } else {
                removed = application.containerCompleted(rmContainer, containerStatus, event, node.getPartition());
                node.releaseContainer(rmContainer.getContainerId(), false);
            }
            // Book-keeping
            if (removed) {
                // Inform the ordering policy
                orderingPolicy.containerReleased(application, rmContainer);
                releaseResource(clusterResource, application, container.getResource(), node.getPartition(), rmContainer);
            }
        } finally {
            writeLock.unlock();
        }
        if (removed) {
            // Inform the parent queue _outside_ of the leaf-queue lock
            getParent().completedContainer(clusterResource, application, node, rmContainer, null, event, this, sortQueues);
        }
    }
    // Notify PreemptionManager
    csContext.getPreemptionManager().removeKillableContainer(new KillableContainer(rmContainer, node.getPartition(), queueName));
}

Example 33 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class CapacityScheduler method completedContainerInternal.

@Override
protected void completedContainerInternal(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) {
    Container container = rmContainer.getContainer();
    ContainerId containerId = container.getId();
    // Get the application for the finished container
    FiCaSchedulerApp application = getCurrentAttemptForContainer(container.getId());
    ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
    if (application == null) {
        LOG.info("Container " + container + " of" + " finished application " + appId + " completed with event " + event);
        return;
    }
    // Get the node on which the container was allocated
    FiCaSchedulerNode node = getNode(container.getNodeId());
    if (null == node) {
        LOG.info("Container " + container + " of" + " removed node " + container.getNodeId() + " completed with event " + event);
        return;
    }
    // Inform the queue
    LeafQueue queue = (LeafQueue) application.getQueue();
    queue.completedContainer(getClusterResource(), application, node, rmContainer, containerStatus, event, null, true);
}

Example 34 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class SchedulerApplicationAttempt method pullNewlyUpdatedContainers.

/**
   * A container is promoted if its executionType is changed from
   * OPPORTUNISTIC to GUARANTEED. It id demoted if the change is from
   * GUARANTEED to OPPORTUNISTIC.
   * @return Newly Promoted and Demoted containers
   */
private List<Container> pullNewlyUpdatedContainers(Map<ContainerId, RMContainer> newlyUpdatedContainers, ContainerUpdateType updateTpe) {
    List<Container> updatedContainers = new ArrayList<>();
    if (oppContainerContext == null && (ContainerUpdateType.DEMOTE_EXECUTION_TYPE == updateTpe || ContainerUpdateType.PROMOTE_EXECUTION_TYPE == updateTpe)) {
        return updatedContainers;
    }
    try {
        writeLock.lock();
        Iterator<Map.Entry<ContainerId, RMContainer>> i = newlyUpdatedContainers.entrySet().iterator();
        while (i.hasNext()) {
            Map.Entry<ContainerId, RMContainer> entry = i.next();
            ContainerId matchedContainerId = entry.getKey();
            RMContainer tempRMContainer = entry.getValue();
            RMContainer existingRMContainer = getRMContainer(matchedContainerId);
            if (existingRMContainer != null) {
                // swap containers
                existingRMContainer = getUpdateContext().swapContainer(tempRMContainer, existingRMContainer, updateTpe);
                getUpdateContext().removeFromOutstandingUpdate(tempRMContainer.getAllocatedSchedulerKey(), existingRMContainer.getContainer());
                Container updatedContainer = updateContainerAndNMToken(existingRMContainer, updateTpe);
                updatedContainers.add(updatedContainer);
            }
            tempContainerToKill.add(tempRMContainer);
            i.remove();
        }
        // Release all temporary containers
        Iterator<RMContainer> tempIter = tempContainerToKill.iterator();
        while (tempIter.hasNext()) {
            RMContainer c = tempIter.next();
            // Mark container for release (set RRs to null, so RM does not think
            // it is a recoverable container)
            ((RMContainerImpl) c).setResourceRequests(null);
            ((AbstractYarnScheduler) rmContext.getScheduler()).completedContainer(c, SchedulerUtils.createAbnormalContainerStatus(c.getContainerId(), SchedulerUtils.UPDATED_CONTAINER), RMContainerEventType.KILL);
            tempIter.remove();
        }
        return updatedContainers;
    } finally {
        writeLock.unlock();
    }
}

Example 35 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class SchedulerApplicationAttempt method pullNewlyAllocatedContainers.

// Create container token and update NMToken altogether, if either of them fails for
// some reason like DNS unavailable, do not return this container and keep it
// in the newlyAllocatedContainers waiting to be refetched.
public List<Container> pullNewlyAllocatedContainers() {
    try {
        writeLock.lock();
        List<Container> returnContainerList = new ArrayList<Container>(newlyAllocatedContainers.size());
        Iterator<RMContainer> i = newlyAllocatedContainers.iterator();
        while (i.hasNext()) {
            RMContainer rmContainer = i.next();
            Container updatedContainer = updateContainerAndNMToken(rmContainer, null);
            // caused by DNS resolving failed.
            if (updatedContainer != null) {
                returnContainerList.add(updatedContainer);
                i.remove();
            }
        }
        return returnContainerList;
    } finally {
        writeLock.unlock();
    }
}