Examples with Container org.apache.hadoop.yarn.api.records.Container used on opensource projects

Example 86 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class FSAppAttempt method assignContainer.

/**
   * Assign a container to this node to facilitate {@code request}. If node does
   * not have enough memory, create a reservation. This is called once we are
   * sure the particular request should be facilitated by this node.
   *
   * @param node
   *     The node to try placing the container on.
   * @param pendingAsk
   *     The {@link PendingAsk} we're trying to satisfy.
   * @param type
   *     The locality of the assignment.
   * @param reserved
   *     Whether there's already a container reserved for this app on the node.
   * @return
   *     If an assignment was made, returns the resources allocated to the
   *     container.  If a reservation was made, returns
   *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
   *     made, returns an empty resource.
   */
private Resource assignContainer(FSSchedulerNode node, PendingAsk pendingAsk, NodeType type, boolean reserved, SchedulerRequestKey schedulerKey) {
    // How much does this request need?
    Resource capability = pendingAsk.getPerAllocationResource();
    // How much does the node have?
    Resource available = node.getUnallocatedResource();
    Container reservedContainer = null;
    if (reserved) {
        reservedContainer = node.getReservedContainer().getContainer();
    }
    // Can we allocate a container on this node?
    if (Resources.fitsIn(capability, available)) {
        // Inform the application of the new container for this request
        RMContainer allocatedContainer = allocate(type, node, schedulerKey, pendingAsk, reservedContainer);
        if (allocatedContainer == null) {
            // Did the application need this resource?
            if (reserved) {
                unreserve(schedulerKey, node);
            }
            return Resources.none();
        }
        // If we had previously made a reservation, delete it
        if (reserved) {
            unreserve(schedulerKey, node);
        }
        // Inform the node
        node.allocateContainer(allocatedContainer);
        // usage
        if (!isAmRunning() && !getUnmanagedAM()) {
            setAMResource(capability);
            getQueue().addAMResourceUsage(capability);
            setAmRunning(true);
        }
        return capability;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Resource request: " + capability + " exceeds the available" + " resources of the node.");
    }
    // The desired container won't fit here, so reserve
    if (isReservable(capability) && reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer, type, schedulerKey)) {
        updateAMDiagnosticMsg(capability, " exceeds the available resources of " + "the node and the request is reserved)");
        if (LOG.isDebugEnabled()) {
            LOG.debug(getName() + "'s resource request is reserved.");
        }
        return FairScheduler.CONTAINER_RESERVED;
    } else {
        updateAMDiagnosticMsg(capability, " exceeds the available resources of " + "the node and the request cannot be reserved)");
        if (LOG.isDebugEnabled()) {
            LOG.debug("Couldn't create reservation for app:  " + getName() + ", at priority " + schedulerKey.getPriority());
        }
        return Resources.none();
    }
}

Example 87 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class FifoScheduler method assignContainer.

private int assignContainer(FiCaSchedulerNode node, FifoAppAttempt application, SchedulerRequestKey schedulerKey, int assignableContainers, Resource capability, NodeType type) {
    LOG.debug("assignContainers:" + " node=" + node.getRMNode().getNodeAddress() + " application=" + application.getApplicationId().getId() + " priority=" + schedulerKey.getPriority().getPriority() + " assignableContainers=" + assignableContainers + " capability=" + capability + " type=" + type);
    // TODO: A buggy application with this zero would crash the scheduler.
    int availableContainers = (int) (node.getUnallocatedResource().getMemorySize() / capability.getMemorySize());
    int assignedContainers = Math.min(assignableContainers, availableContainers);
    if (assignedContainers > 0) {
        for (int i = 0; i < assignedContainers; ++i) {
            NodeId nodeId = node.getRMNode().getNodeID();
            ContainerId containerId = BuilderUtils.newContainerId(application.getApplicationAttemptId(), application.getNewContainerId());
            // Create the container
            Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode().getHttpAddress(), capability, schedulerKey.getPriority(), null, schedulerKey.getAllocationRequestId());
            // Allocate!
            // Inform the application
            RMContainer rmContainer = application.allocate(type, node, schedulerKey, container);
            // Inform the node
            node.allocateContainer(rmContainer);
            // Update usage for this container
            increaseUsedResources(rmContainer);
        }
    }
    return assignedContainers;
}

Example 88 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class FifoScheduler method completedContainerInternal.

@Lock(FifoScheduler.class)
@Override
protected synchronized void completedContainerInternal(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) {
    // Get the application for the finished container
    Container container = rmContainer.getContainer();
    FifoAppAttempt application = getCurrentAttemptForContainer(container.getId());
    ApplicationId appId = container.getId().getApplicationAttemptId().getApplicationId();
    // Get the node on which the container was allocated
    FiCaSchedulerNode node = (FiCaSchedulerNode) getNode(container.getNodeId());
    if (application == null) {
        LOG.info("Unknown application: " + appId + " released container " + container.getId() + " on node: " + node + " with event: " + event);
        return;
    }
    // Inform the application
    application.containerCompleted(rmContainer, containerStatus, event, RMNodeLabelsManager.NO_LABEL);
    // Inform the node
    node.releaseContainer(rmContainer.getContainerId(), false);
    // Update total usage
    Resources.subtractFrom(usedResource, container.getResource());
    LOG.info("Application attempt " + application.getApplicationAttemptId() + " released container " + container.getId() + " on node: " + node + " with event: " + event);
}

Example 89 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class FairScheduler method allocate.

@Override
public Allocation allocate(ApplicationAttemptId appAttemptId, List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests) {
    // Make sure this application exists
    FSAppAttempt application = getSchedulerApp(appAttemptId);
    if (application == null) {
        LOG.info("Calling allocate on removed " + "or non existent application " + appAttemptId);
        return EMPTY_ALLOCATION;
    }
    // Handle promotions and demotions
    handleContainerUpdates(application, updateRequests);
    // Sanity check
    normalizeRequests(ask);
    // Record container allocation start time
    application.recordContainerRequestTime(getClock().getTime());
    // Release containers
    releaseContainers(release, application);
    ReentrantReadWriteLock.WriteLock lock = application.getWriteLock();
    lock.lock();
    try {
        if (!ask.isEmpty()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("allocate: pre-update" + " applicationAttemptId=" + appAttemptId + " application=" + application.getApplicationId());
            }
            application.showRequests();
            // Update application requests
            application.updateResourceRequests(ask);
            application.showRequests();
        }
    } finally {
        lock.unlock();
    }
    Set<ContainerId> preemptionContainerIds = application.getPreemptionContainerIds();
    if (LOG.isDebugEnabled()) {
        LOG.debug("allocate: post-update" + " applicationAttemptId=" + appAttemptId + " #ask=" + ask.size() + " reservation= " + application.getCurrentReservation());
        LOG.debug("Preempting " + preemptionContainerIds.size() + " container(s)");
    }
    application.updateBlacklist(blacklistAdditions, blacklistRemovals);
    List<Container> newlyAllocatedContainers = application.pullNewlyAllocatedContainers();
    // Record container allocation time
    if (!(newlyAllocatedContainers.isEmpty())) {
        application.recordContainerAllocationTime(getClock().getTime());
    }
    Resource headroom = application.getHeadroom();
    application.setApplicationHeadroomForMetrics(headroom);
    return new Allocation(newlyAllocatedContainers, headroom, preemptionContainerIds, null, null, application.pullUpdatedNMTokens(), null, null, application.pullNewlyPromotedContainers(), application.pullNewlyDemotedContainers());
}

Example 90 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class AMRMClientImpl method allocate.

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();
    Map<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> oldChange = new HashMap<>();
    try {
        synchronized (this) {
            askList = cloneAsks();
            // Save the current change for recovery
            oldChange.putAll(change);
            List<UpdateContainerRequest> updateList = createUpdateList();
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();
            change.clear();
            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove);
            allocateRequest = AllocateRequest.newBuilder().responseId(lastResponseId).progress(progressIndicator).askList(askList).resourceBlacklistRequest(blacklistRequest).releaseList(releaseList).updateRequests(updateList).build();
            // clear blacklistAdditions and blacklistRemovals before
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }
        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (RemoteRequestsTable remoteRequestsTable : remoteRequests.values()) {
                    @SuppressWarnings("unchecked") Iterator<ResourceRequestInfo<T>> reqIter = remoteRequestsTable.iterator();
                    while (reqIter.hasNext()) {
                        addResourceRequestToAsk(reqIter.next().remoteRequest);
                    }
                }
                change.putAll(this.pendingChange);
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }
        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
            if (!pendingChange.isEmpty()) {
                List<ContainerStatus> completed = allocateResponse.getCompletedContainersStatuses();
                List<UpdatedContainer> changed = new ArrayList<>();
                changed.addAll(allocateResponse.getUpdatedContainers());
                // containers
                for (ContainerStatus status : completed) {
                    ContainerId containerId = status.getContainerId();
                    pendingChange.remove(containerId);
                }
                // remove all pending change requests that have been satisfied
                if (!changed.isEmpty()) {
                    removePendingChangeRequests(changed);
                }
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }
                // that do not exist in the current change map:
                for (Map.Entry<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> entry : oldChange.entrySet()) {
                    ContainerId oldContainerId = entry.getKey();
                    Container oldContainer = entry.getValue().getKey();
                    UpdateContainerRequest oldupdate = entry.getValue().getValue();
                    if (change.get(oldContainerId) == null) {
                        change.put(oldContainerId, new SimpleEntry<>(oldContainer, oldupdate));
                    }
                }
                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}