Examples with RMContainer org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer used on opensource projects

Example 66 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FSAppAttempt method allocate.

public RMContainer allocate(NodeType type, FSSchedulerNode node, SchedulerRequestKey schedulerKey, PendingAsk pendingAsk, Container reservedContainer) {
    RMContainer rmContainer;
    Container container;
    try {
        writeLock.lock();
        // Update allowed locality level
        NodeType allowed = allowedLocalityLevel.get(schedulerKey);
        if (allowed != null) {
            if (allowed.equals(NodeType.OFF_SWITCH) && (type.equals(NodeType.NODE_LOCAL) || type.equals(NodeType.RACK_LOCAL))) {
                this.resetAllowedLocalityLevel(schedulerKey, type);
            } else if (allowed.equals(NodeType.RACK_LOCAL) && type.equals(NodeType.NODE_LOCAL)) {
                this.resetAllowedLocalityLevel(schedulerKey, type);
            }
        }
        // request without locking the scheduler, hence we need to check
        if (getOutstandingAsksCount(schedulerKey) <= 0) {
            return null;
        }
        container = reservedContainer;
        if (container == null) {
            container = createContainer(node, pendingAsk.getPerAllocationResource(), schedulerKey);
        }
        // Create RMContainer
        rmContainer = new RMContainerImpl(container, schedulerKey, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
        ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
        // Add it to allContainers list.
        addToNewlyAllocatedContainers(node, rmContainer);
        liveContainers.put(container.getId(), rmContainer);
        // Update consumption and track allocations
        List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(type, node, schedulerKey, container);
        this.attemptResourceUsage.incUsed(container.getResource());
        // Update resource requests related to "request" and store in RMContainer
        ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
        // Inform the container
        rmContainer.handle(new RMContainerEvent(container.getId(), RMContainerEventType.START));
        if (LOG.isDebugEnabled()) {
            LOG.debug("allocate: applicationAttemptId=" + container.getId().getApplicationAttemptId() + " container=" + container.getId() + " host=" + container.getNodeId().getHost() + " type=" + type);
        }
        RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), container.getId(), container.getResource());
    } finally {
        writeLock.unlock();
    }
    return rmContainer;
}

Example 67 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FSAppAttempt method assignContainer.

/**
   * Assign a container to this node to facilitate {@code request}. If node does
   * not have enough memory, create a reservation. This is called once we are
   * sure the particular request should be facilitated by this node.
   *
   * @param node
   *     The node to try placing the container on.
   * @param pendingAsk
   *     The {@link PendingAsk} we're trying to satisfy.
   * @param type
   *     The locality of the assignment.
   * @param reserved
   *     Whether there's already a container reserved for this app on the node.
   * @return
   *     If an assignment was made, returns the resources allocated to the
   *     container.  If a reservation was made, returns
   *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
   *     made, returns an empty resource.
   */
private Resource assignContainer(FSSchedulerNode node, PendingAsk pendingAsk, NodeType type, boolean reserved, SchedulerRequestKey schedulerKey) {
    // How much does this request need?
    Resource capability = pendingAsk.getPerAllocationResource();
    // How much does the node have?
    Resource available = node.getUnallocatedResource();
    Container reservedContainer = null;
    if (reserved) {
        reservedContainer = node.getReservedContainer().getContainer();
    }
    // Can we allocate a container on this node?
    if (Resources.fitsIn(capability, available)) {
        // Inform the application of the new container for this request
        RMContainer allocatedContainer = allocate(type, node, schedulerKey, pendingAsk, reservedContainer);
        if (allocatedContainer == null) {
            // Did the application need this resource?
            if (reserved) {
                unreserve(schedulerKey, node);
            }
            return Resources.none();
        }
        // If we had previously made a reservation, delete it
        if (reserved) {
            unreserve(schedulerKey, node);
        }
        // Inform the node
        node.allocateContainer(allocatedContainer);
        // usage
        if (!isAmRunning() && !getUnmanagedAM()) {
            setAMResource(capability);
            getQueue().addAMResourceUsage(capability);
            setAmRunning(true);
        }
        return capability;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Resource request: " + capability + " exceeds the available" + " resources of the node.");
    }
    // The desired container won't fit here, so reserve
    if (isReservable(capability) && reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer, type, schedulerKey)) {
        updateAMDiagnosticMsg(capability, " exceeds the available resources of " + "the node and the request is reserved)");
        if (LOG.isDebugEnabled()) {
            LOG.debug(getName() + "'s resource request is reserved.");
        }
        return FairScheduler.CONTAINER_RESERVED;
    } else {
        updateAMDiagnosticMsg(capability, " exceeds the available resources of " + "the node and the request cannot be reserved)");
        if (LOG.isDebugEnabled()) {
            LOG.debug("Couldn't create reservation for app:  " + getName() + ", at priority " + schedulerKey.getPriority());
        }
        return Resources.none();
    }
}

Example 68 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FSAppAttempt method assignReservedContainer.

/**
   * Called when this application already has an existing reservation on the
   * given node.  Sees whether we can turn the reservation into an allocation.
   * Also checks whether the application needs the reservation anymore, and
   * releases it if not.
   *
   * @param node
   *     Node that the application has an existing reservation on
   * @return whether the reservation on the given node is valid.
   */
boolean assignReservedContainer(FSSchedulerNode node) {
    RMContainer rmContainer = node.getReservedContainer();
    SchedulerRequestKey reservedSchedulerKey = rmContainer.getReservedSchedulerKey();
    if (!isValidReservation(node)) {
        // Don't hold the reservation if app can no longer use it
        LOG.info("Releasing reservation that cannot be satisfied for " + "application " + getApplicationAttemptId() + " on node " + node);
        unreserve(reservedSchedulerKey, node);
        return false;
    }
    // Reservation valid; try to fulfill the reservation
    if (LOG.isDebugEnabled()) {
        LOG.debug("Trying to fulfill reservation for application " + getApplicationAttemptId() + " on node: " + node);
    }
    // there's only one container size per priority.
    if (Resources.fitsIn(node.getReservedContainer().getReservedResource(), node.getUnallocatedResource())) {
        assignContainer(node, true);
    }
    return true;
}

Example 69 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FSAppAttempt method unreserve.

/**
   * Remove the reservation on {@code node} at the given SchedulerRequestKey.
   * This dispatches SchedulerNode handlers as well.
   * @param schedulerKey Scheduler Key
   * @param node Node
   */
public void unreserve(SchedulerRequestKey schedulerKey, FSSchedulerNode node) {
    RMContainer rmContainer = node.getReservedContainer();
    unreserveInternal(schedulerKey, node);
    node.unreserveResource(this);
    clearReservation(node);
    getMetrics().unreserveResource(getUser(), rmContainer.getContainer().getResource());
}

Example 70 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FifoScheduler method assignContainer.

private int assignContainer(FiCaSchedulerNode node, FifoAppAttempt application, SchedulerRequestKey schedulerKey, int assignableContainers, Resource capability, NodeType type) {
    LOG.debug("assignContainers:" + " node=" + node.getRMNode().getNodeAddress() + " application=" + application.getApplicationId().getId() + " priority=" + schedulerKey.getPriority().getPriority() + " assignableContainers=" + assignableContainers + " capability=" + capability + " type=" + type);
    // TODO: A buggy application with this zero would crash the scheduler.
    int availableContainers = (int) (node.getUnallocatedResource().getMemorySize() / capability.getMemorySize());
    int assignedContainers = Math.min(assignableContainers, availableContainers);
    if (assignedContainers > 0) {
        for (int i = 0; i < assignedContainers; ++i) {
            NodeId nodeId = node.getRMNode().getNodeID();
            ContainerId containerId = BuilderUtils.newContainerId(application.getApplicationAttemptId(), application.getNewContainerId());
            // Create the container
            Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode().getHttpAddress(), capability, schedulerKey.getPriority(), null, schedulerKey.getAllocationRequestId());
            // Allocate!
            // Inform the application
            RMContainer rmContainer = application.allocate(type, node, schedulerKey, container);
            // Inform the node
            node.allocateContainer(rmContainer);
            // Update usage for this container
            increaseUsedResources(rmContainer);
        }
    }
    return assignedContainers;
}