Examples with NodeId org.apache.hadoop.yarn.api.records.NodeId used on opensource projects

Example 36 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class ContainerUpdateContext method matchContainerToOutstandingIncreaseReq.

/**
   * Check if a new container is to be matched up against an outstanding
   * Container increase request.
   * @param node SchedulerNode.
   * @param schedulerKey SchedulerRequestKey.
   * @param rmContainer RMContainer.
   * @return ContainerId.
   */
public ContainerId matchContainerToOutstandingIncreaseReq(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer) {
    ContainerId retVal = null;
    Container container = rmContainer.getContainer();
    Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap = outstandingIncreases.get(schedulerKey);
    if (resourceMap != null) {
        Map<NodeId, Set<ContainerId>> locationMap = resourceMap.get(container.getResource());
        if (locationMap != null) {
            Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
            if (containerIds != null && !containerIds.isEmpty()) {
                retVal = containerIds.iterator().next();
            }
        }
    }
    // We also need to add these requests back.. to be reallocated.
    if (resourceMap != null && retVal == null) {
        Map<SchedulerRequestKey, Map<String, ResourceRequest>> reqsToUpdate = new HashMap<>();
        Map<String, ResourceRequest> resMap = createResourceRequests(rmContainer, node, schedulerKey, rmContainer.getContainer().getResource());
        reqsToUpdate.put(schedulerKey, resMap);
        appSchedulingInfo.addToPlacementSets(true, reqsToUpdate);
        return UNDEFINED;
    }
    return retVal;
}

Example 37 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class ContainerUpdateContext method checkAndAddToOutstandingIncreases.

/**
   * Add the container to outstanding increases.
   * @param rmContainer RMContainer.
   * @param schedulerNode SchedulerNode.
   * @param updateRequest UpdateContainerRequest.
   * @return true if updated to outstanding increases was successful.
   */
public synchronized boolean checkAndAddToOutstandingIncreases(RMContainer rmContainer, SchedulerNode schedulerNode, UpdateContainerRequest updateRequest) {
    Container container = rmContainer.getContainer();
    SchedulerRequestKey schedulerKey = SchedulerRequestKey.create(updateRequest, rmContainer.getAllocatedSchedulerKey());
    Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap = outstandingIncreases.get(schedulerKey);
    if (resourceMap == null) {
        resourceMap = new HashMap<>();
        outstandingIncreases.put(schedulerKey, resourceMap);
    } else {
        // Updating Resource for and existing increase container
        if (ContainerUpdateType.INCREASE_RESOURCE == updateRequest.getContainerUpdateType()) {
            cancelPreviousRequest(schedulerNode, schedulerKey);
        } else {
            return false;
        }
    }
    Resource resToIncrease = getResourceToIncrease(updateRequest, rmContainer);
    Map<NodeId, Set<ContainerId>> locationMap = resourceMap.get(resToIncrease);
    if (locationMap == null) {
        locationMap = new HashMap<>();
        resourceMap.put(resToIncrease, locationMap);
    }
    Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
    if (containerIds == null) {
        containerIds = new HashSet<>();
        locationMap.put(container.getNodeId(), containerIds);
    }
    if (outstandingDecreases.containsKey(container.getId())) {
        return false;
    }
    containerIds.add(container.getId());
    if (!Resources.isNone(resToIncrease)) {
        Map<SchedulerRequestKey, Map<String, ResourceRequest>> updateResReqs = new HashMap<>();
        Map<String, ResourceRequest> resMap = createResourceRequests(rmContainer, schedulerNode, schedulerKey, resToIncrease);
        updateResReqs.put(schedulerKey, resMap);
        appSchedulingInfo.addToPlacementSets(false, updateResReqs);
    }
    return true;
}

Example 38 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class SchedulerApplicationAttempt method commonReserve.

private boolean commonReserve(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer, Resource reservedResource) {
    try {
        rmContainer.handle(new RMContainerReservedEvent(rmContainer.getContainerId(), reservedResource, node.getNodeID(), schedulerKey));
    } catch (InvalidStateTransitionException e) {
        // false indicate it fails
        return false;
    }
    Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(schedulerKey);
    if (reservedContainers == null) {
        reservedContainers = new HashMap<NodeId, RMContainer>();
        this.reservedContainers.put(schedulerKey, reservedContainers);
    }
    reservedContainers.put(node.getNodeID(), rmContainer);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Application attempt " + getApplicationAttemptId() + " reserved container " + rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers.size() + " reserved containers at priority " + schedulerKey.getPriority() + "; currentReservation " + reservedResource);
    }
    return true;
}

Example 39 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class SchedulerApplicationAttempt method move.

public void move(Queue newQueue) {
    try {
        writeLock.lock();
        QueueMetrics oldMetrics = queue.getMetrics();
        QueueMetrics newMetrics = newQueue.getMetrics();
        String newQueueName = newQueue.getQueueName();
        String user = getUser();
        for (RMContainer liveContainer : liveContainers.values()) {
            Resource resource = liveContainer.getContainer().getResource();
            ((RMContainerImpl) liveContainer).setQueueName(newQueueName);
            oldMetrics.releaseResources(user, 1, resource);
            newMetrics.allocateResources(user, 1, resource, false);
        }
        for (Map<NodeId, RMContainer> map : reservedContainers.values()) {
            for (RMContainer reservedContainer : map.values()) {
                ((RMContainerImpl) reservedContainer).setQueueName(newQueueName);
                Resource resource = reservedContainer.getReservedResource();
                oldMetrics.unreserveResource(user, resource);
                newMetrics.reserveResource(user, resource);
            }
        }
        if (!isStopped) {
            appSchedulingInfo.move(newQueue);
        }
        this.queue = newQueue;
    } finally {
        writeLock.unlock();
    }
}

Example 40 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class FiCaSchedulerApp method internalUnreserve.

private boolean internalUnreserve(FiCaSchedulerNode node, SchedulerRequestKey schedulerKey) {
    Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(schedulerKey);
    if (reservedContainers != null) {
        RMContainer reservedContainer = reservedContainers.remove(node.getNodeID());
        // as a consequence reservedcontainer might be null, adding NP-checks
        if (reservedContainer != null && reservedContainer.getContainer() != null && reservedContainer.getContainer().getResource() != null) {
            if (reservedContainers.isEmpty()) {
                this.reservedContainers.remove(schedulerKey);
            }
            // Reset the re-reservation count
            resetReReservations(schedulerKey);
            Resource resource = reservedContainer.getReservedResource();
            this.attemptResourceUsage.decReserved(node.getPartition(), resource);
            LOG.info("Application " + getApplicationId() + " unreserved " + " on node " + node + ", currently has " + reservedContainers.size() + " at priority " + schedulerKey.getPriority() + "; currentReservation " + this.attemptResourceUsage.getReserved() + " on node-label=" + node.getPartition());
            return true;
        }
    }
    return false;
}