Examples with RMContainer org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer used on opensource projects

Example 31 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class CapacityScheduler method completedContainerInternal.

@Override
protected void completedContainerInternal(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) {
    Container container = rmContainer.getContainer();
    ContainerId containerId = container.getId();
    // Get the application for the finished container
    FiCaSchedulerApp application = getCurrentAttemptForContainer(container.getId());
    ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
    if (application == null) {
        LOG.info("Container " + container + " of" + " finished application " + appId + " completed with event " + event);
        return;
    }
    // Get the node on which the container was allocated
    FiCaSchedulerNode node = getNode(container.getNodeId());
    if (null == node) {
        LOG.info("Container " + container + " of" + " removed node " + container.getNodeId() + " completed with event " + event);
        return;
    }
    // Inform the queue
    LeafQueue queue = (LeafQueue) application.getQueue();
    queue.completedContainer(getClusterResource(), application, node, rmContainer, containerStatus, event, null, true);
}

Example 32 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class CapacityScheduler method allocateContainerOnSingleNode.

/*
   * Logics of allocate container on a single node (Old behavior)
   */
private CSAssignment allocateContainerOnSingleNode(PlacementSet<FiCaSchedulerNode> ps, FiCaSchedulerNode node, boolean withNodeHeartbeat) {
    // driven by node heartbeat works.
    if (getNode(node.getNodeID()) != node) {
        LOG.error("Trying to schedule on a removed node, please double check.");
        return null;
    }
    CSAssignment assignment;
    // Assign new containers...
    // 1. Check for reserved applications
    // 2. Schedule if there are no reservations
    RMContainer reservedContainer = node.getReservedContainer();
    if (reservedContainer != null) {
        FiCaSchedulerApp reservedApplication = getCurrentAttemptForContainer(reservedContainer.getContainerId());
        // Try to fulfill the reservation
        LOG.info("Trying to fulfill reservation for application " + reservedApplication.getApplicationId() + " on node: " + node.getNodeID());
        LeafQueue queue = ((LeafQueue) reservedApplication.getQueue());
        assignment = queue.assignContainers(getClusterResource(), ps, // resources, should consider labeled resources as well.
        new ResourceLimits(labelManager.getResourceByLabel(RMNodeLabelsManager.NO_LABEL, getClusterResource())), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
        if (assignment.isFulfilledReservation()) {
            if (withNodeHeartbeat) {
                // Only update SchedulerHealth in sync scheduling, existing
                // Data structure of SchedulerHealth need to be updated for
                // Async mode
                updateSchedulerHealth(lastNodeUpdateTime, node.getNodeID(), assignment);
            }
            schedulerHealth.updateSchedulerFulfilledReservationCounts(1);
            ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, queue.getParent().getQueueName(), queue.getQueueName(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY);
            ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, node, reservedContainer.getContainerId(), AllocationState.ALLOCATED_FROM_RESERVED);
        } else {
            ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, queue.getParent().getQueueName(), queue.getQueueName(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY);
            ActivitiesLogger.NODE.finishAllocatedNodeAllocation(activitiesManager, node, reservedContainer.getContainerId(), AllocationState.SKIPPED);
        }
        assignment.setSchedulingMode(SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
        submitResourceCommitRequest(getClusterResource(), assignment);
    }
    // Do not schedule if there are any reservations to fulfill on the node
    if (node.getReservedContainer() != null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Skipping scheduling since node " + node.getNodeID() + " is reserved by application " + node.getReservedContainer().getContainerId().getApplicationAttemptId());
        }
        return null;
    }
    // has any available or killable resource
    if (calculator.computeAvailableContainers(Resources.add(node.getUnallocatedResource(), node.getTotalKillableResources()), minimumAllocation) <= 0) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("This node or this node partition doesn't have available or" + "killable resource");
        }
        return null;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Trying to schedule on node: " + node.getNodeName() + ", available: " + node.getUnallocatedResource());
    }
    return allocateOrReserveNewContainers(ps, withNodeHeartbeat);
}

Example 33 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class CapacityScheduler method moveApplication.

@Override
public String moveApplication(ApplicationId appId, String targetQueueName) throws YarnException {
    try {
        writeLock.lock();
        SchedulerApplication<FiCaSchedulerApp> application = applications.get(appId);
        if (application == null) {
            throw new YarnException("App to be moved " + appId + " not found.");
        }
        String sourceQueueName = application.getQueue().getQueueName();
        LeafQueue source = this.queueManager.getAndCheckLeafQueue(sourceQueueName);
        String destQueueName = handleMoveToPlanQueue(targetQueueName);
        LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName);
        String user = application.getUser();
        try {
            dest.submitApplication(appId, user, destQueueName);
        } catch (AccessControlException e) {
            throw new YarnException(e);
        }
        FiCaSchedulerApp app = application.getCurrentAppAttempt();
        if (app != null) {
            // For transferStateFromPreviousAttempt required
            for (RMContainer rmContainer : app.getLiveContainers()) {
                source.detachContainer(getClusterResource(), app, rmContainer);
                // attach the Container to another queue
                dest.attachContainer(getClusterResource(), app, rmContainer);
            }
            if (!app.isStopped()) {
                source.finishApplicationAttempt(app, sourceQueueName);
                // Submit to a new queue
                dest.submitApplicationAttempt(app, user);
            }
            // Finish app & update metrics
            app.move(dest);
        }
        source.appFinished();
        // Detach the application..
        source.getParent().finishApplication(appId, user);
        application.setQueue(dest);
        LOG.info("App: " + appId + " successfully moved from " + sourceQueueName + " to: " + destQueueName);
        return targetQueueName;
    } finally {
        writeLock.unlock();
    }
}

Example 34 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SchedulerApplicationAttempt method commonReserve.

private boolean commonReserve(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer, Resource reservedResource) {
    try {
        rmContainer.handle(new RMContainerReservedEvent(rmContainer.getContainerId(), reservedResource, node.getNodeID(), schedulerKey));
    } catch (InvalidStateTransitionException e) {
        // false indicate it fails
        return false;
    }
    Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(schedulerKey);
    if (reservedContainers == null) {
        reservedContainers = new HashMap<NodeId, RMContainer>();
        this.reservedContainers.put(schedulerKey, reservedContainers);
    }
    reservedContainers.put(node.getNodeID(), rmContainer);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Application attempt " + getApplicationAttemptId() + " reserved container " + rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers.size() + " reserved containers at priority " + schedulerKey.getPriority() + "; currentReservation " + reservedResource);
    }
    return true;
}

Example 35 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SchedulerApplicationAttempt method pullNewlyUpdatedContainers.

/**
   * A container is promoted if its executionType is changed from
   * OPPORTUNISTIC to GUARANTEED. It id demoted if the change is from
   * GUARANTEED to OPPORTUNISTIC.
   * @return Newly Promoted and Demoted containers
   */
private List<Container> pullNewlyUpdatedContainers(Map<ContainerId, RMContainer> newlyUpdatedContainers, ContainerUpdateType updateTpe) {
    List<Container> updatedContainers = new ArrayList<>();
    if (oppContainerContext == null && (ContainerUpdateType.DEMOTE_EXECUTION_TYPE == updateTpe || ContainerUpdateType.PROMOTE_EXECUTION_TYPE == updateTpe)) {
        return updatedContainers;
    }
    try {
        writeLock.lock();
        Iterator<Map.Entry<ContainerId, RMContainer>> i = newlyUpdatedContainers.entrySet().iterator();
        while (i.hasNext()) {
            Map.Entry<ContainerId, RMContainer> entry = i.next();
            ContainerId matchedContainerId = entry.getKey();
            RMContainer tempRMContainer = entry.getValue();
            RMContainer existingRMContainer = getRMContainer(matchedContainerId);
            if (existingRMContainer != null) {
                // swap containers
                existingRMContainer = getUpdateContext().swapContainer(tempRMContainer, existingRMContainer, updateTpe);
                getUpdateContext().removeFromOutstandingUpdate(tempRMContainer.getAllocatedSchedulerKey(), existingRMContainer.getContainer());
                Container updatedContainer = updateContainerAndNMToken(existingRMContainer, updateTpe);
                updatedContainers.add(updatedContainer);
            }
            tempContainerToKill.add(tempRMContainer);
            i.remove();
        }
        // Release all temporary containers
        Iterator<RMContainer> tempIter = tempContainerToKill.iterator();
        while (tempIter.hasNext()) {
            RMContainer c = tempIter.next();
            // Mark container for release (set RRs to null, so RM does not think
            // it is a recoverable container)
            ((RMContainerImpl) c).setResourceRequests(null);
            ((AbstractYarnScheduler) rmContext.getScheduler()).completedContainer(c, SchedulerUtils.createAbnormalContainerStatus(c.getContainerId(), SchedulerUtils.UPDATED_CONTAINER), RMContainerEventType.KILL);
            tempIter.remove();
        }
        return updatedContainers;
    } finally {
        writeLock.unlock();
    }
}