Examples with RMContainer org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer used on opensource projects

Example 26 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class CapacityScheduler method moveApplication.

@Override
public String moveApplication(ApplicationId appId, String targetQueueName) throws YarnException {
    try {
        writeLock.lock();
        SchedulerApplication<FiCaSchedulerApp> application = applications.get(appId);
        if (application == null) {
            throw new YarnException("App to be moved " + appId + " not found.");
        }
        String sourceQueueName = application.getQueue().getQueueName();
        LeafQueue source = this.queueManager.getAndCheckLeafQueue(sourceQueueName);
        String destQueueName = handleMoveToPlanQueue(targetQueueName);
        LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName);
        String user = application.getUser();
        try {
            dest.submitApplication(appId, user, destQueueName);
        } catch (AccessControlException e) {
            throw new YarnException(e);
        }
        FiCaSchedulerApp app = application.getCurrentAppAttempt();
        if (app != null) {
            // For transferStateFromPreviousAttempt required
            for (RMContainer rmContainer : app.getLiveContainers()) {
                source.detachContainer(getClusterResource(), app, rmContainer);
                // attach the Container to another queue
                dest.attachContainer(getClusterResource(), app, rmContainer);
            }
            if (!app.isStopped()) {
                source.finishApplicationAttempt(app, sourceQueueName);
                // Submit to a new queue
                dest.submitApplicationAttempt(app, user);
            }
            // Finish app & update metrics
            app.move(dest);
        }
        source.appFinished();
        // Detach the application..
        source.getParent().finishApplication(appId, user);
        application.setQueue(dest);
        LOG.info("App: " + appId + " successfully moved from " + sourceQueueName + " to: " + destQueueName);
        return targetQueueName;
    } finally {
        writeLock.unlock();
    }
}

Example 27 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SchedulerApplicationAttempt method commonReserve.

private boolean commonReserve(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer, Resource reservedResource) {
    try {
        rmContainer.handle(new RMContainerReservedEvent(rmContainer.getContainerId(), reservedResource, node.getNodeID(), schedulerKey));
    } catch (InvalidStateTransitionException e) {
        // false indicate it fails
        return false;
    }
    Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(schedulerKey);
    if (reservedContainers == null) {
        reservedContainers = new HashMap<NodeId, RMContainer>();
        this.reservedContainers.put(schedulerKey, reservedContainers);
    }
    reservedContainers.put(node.getNodeID(), rmContainer);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Application attempt " + getApplicationAttemptId() + " reserved container " + rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers.size() + " reserved containers at priority " + schedulerKey.getPriority() + "; currentReservation " + reservedResource);
    }
    return true;
}

Example 28 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SchedulerApplicationAttempt method pullNewlyUpdatedContainers.

/**
   * A container is promoted if its executionType is changed from
   * OPPORTUNISTIC to GUARANTEED. It id demoted if the change is from
   * GUARANTEED to OPPORTUNISTIC.
   * @return Newly Promoted and Demoted containers
   */
private List<Container> pullNewlyUpdatedContainers(Map<ContainerId, RMContainer> newlyUpdatedContainers, ContainerUpdateType updateTpe) {
    List<Container> updatedContainers = new ArrayList<>();
    if (oppContainerContext == null && (ContainerUpdateType.DEMOTE_EXECUTION_TYPE == updateTpe || ContainerUpdateType.PROMOTE_EXECUTION_TYPE == updateTpe)) {
        return updatedContainers;
    }
    try {
        writeLock.lock();
        Iterator<Map.Entry<ContainerId, RMContainer>> i = newlyUpdatedContainers.entrySet().iterator();
        while (i.hasNext()) {
            Map.Entry<ContainerId, RMContainer> entry = i.next();
            ContainerId matchedContainerId = entry.getKey();
            RMContainer tempRMContainer = entry.getValue();
            RMContainer existingRMContainer = getRMContainer(matchedContainerId);
            if (existingRMContainer != null) {
                // swap containers
                existingRMContainer = getUpdateContext().swapContainer(tempRMContainer, existingRMContainer, updateTpe);
                getUpdateContext().removeFromOutstandingUpdate(tempRMContainer.getAllocatedSchedulerKey(), existingRMContainer.getContainer());
                Container updatedContainer = updateContainerAndNMToken(existingRMContainer, updateTpe);
                updatedContainers.add(updatedContainer);
            }
            tempContainerToKill.add(tempRMContainer);
            i.remove();
        }
        // Release all temporary containers
        Iterator<RMContainer> tempIter = tempContainerToKill.iterator();
        while (tempIter.hasNext()) {
            RMContainer c = tempIter.next();
            // Mark container for release (set RRs to null, so RM does not think
            // it is a recoverable container)
            ((RMContainerImpl) c).setResourceRequests(null);
            ((AbstractYarnScheduler) rmContext.getScheduler()).completedContainer(c, SchedulerUtils.createAbnormalContainerStatus(c.getContainerId(), SchedulerUtils.UPDATED_CONTAINER), RMContainerEventType.KILL);
            tempIter.remove();
        }
        return updatedContainers;
    } finally {
        writeLock.unlock();
    }
}

Example 29 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SchedulerApplicationAttempt method getRunningAggregateAppResourceUsage.

private AggregateAppResourceUsage getRunningAggregateAppResourceUsage() {
    long currentTimeMillis = System.currentTimeMillis();
    // recently.
    if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
        long memorySeconds = 0;
        long vcoreSeconds = 0;
        for (RMContainer rmContainer : this.liveContainers.values()) {
            long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
            Resource resource = rmContainer.getContainer().getResource();
            memorySeconds += resource.getMemorySize() * usedMillis / DateUtils.MILLIS_PER_SECOND;
            vcoreSeconds += resource.getVirtualCores() * usedMillis / DateUtils.MILLIS_PER_SECOND;
        }
        lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
        lastMemorySeconds = memorySeconds;
        lastVcoreSeconds = vcoreSeconds;
    }
    return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds);
}

Example 30 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SchedulerApplicationAttempt method pullNewlyAllocatedContainers.

// Create container token and update NMToken altogether, if either of them fails for
// some reason like DNS unavailable, do not return this container and keep it
// in the newlyAllocatedContainers waiting to be refetched.
public List<Container> pullNewlyAllocatedContainers() {
    try {
        writeLock.lock();
        List<Container> returnContainerList = new ArrayList<Container>(newlyAllocatedContainers.size());
        Iterator<RMContainer> i = newlyAllocatedContainers.iterator();
        while (i.hasNext()) {
            RMContainer rmContainer = i.next();
            Container updatedContainer = updateContainerAndNMToken(rmContainer, null);
            // caused by DNS resolving failed.
            if (updatedContainer != null) {
                returnContainerList.add(updatedContainer);
                i.remove();
            }
        }
        return returnContainerList;
    } finally {
        writeLock.unlock();
    }
}