Examples with SchedulerRequestKey org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey used on opensource projects

Example 1 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class Application method addTask.

public synchronized void addTask(Task task) {
    SchedulerRequestKey schedulerKey = task.getSchedulerKey();
    Map<String, ResourceRequest> requests = this.requests.get(schedulerKey);
    if (requests == null) {
        requests = new HashMap<String, ResourceRequest>();
        this.requests.put(schedulerKey, requests);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Added priority=" + schedulerKey.getPriority() + " application=" + applicationId);
        }
    }
    final Resource capability = requestSpec.get(schedulerKey);
    // Note down the task
    Set<Task> tasks = this.tasks.get(schedulerKey);
    if (tasks == null) {
        tasks = new HashSet<Task>();
        this.tasks.put(schedulerKey, tasks);
    }
    tasks.add(task);
    LOG.info("Added task " + task.getTaskId() + " to application " + applicationId + " at priority " + schedulerKey.getPriority());
    if (LOG.isDebugEnabled()) {
        LOG.debug("addTask: application=" + applicationId + " #asks=" + ask.size());
    }
    // Create resource requests
    for (String host : task.getHosts()) {
        // Data-local
        addResourceRequest(schedulerKey, requests, host, capability);
    }
    // Rack-local
    for (String rack : task.getRacks()) {
        addResourceRequest(schedulerKey, requests, rack, capability);
    }
    // Off-switch
    addResourceRequest(schedulerKey, requests, ResourceRequest.ANY, capability);
}

Example 2 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class Application method assign.

public synchronized void assign(List<Container> containers) throws IOException, YarnException {
    int numContainers = containers.size();
    // Schedule in priority order
    for (SchedulerRequestKey schedulerKey : requests.keySet()) {
        assign(schedulerKey, NodeType.NODE_LOCAL, containers);
        assign(schedulerKey, NodeType.RACK_LOCAL, containers);
        assign(schedulerKey, NodeType.OFF_SWITCH, containers);
        if (containers.isEmpty()) {
            break;
        }
    }
    int assignedContainers = numContainers - containers.size();
    LOG.info("Application " + applicationId + " assigned " + assignedContainers + "/" + numContainers);
}

Example 3 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class ContainerUpdateContext method matchContainerToOutstandingIncreaseReq.

/**
   * Check if a new container is to be matched up against an outstanding
   * Container increase request.
   * @param node SchedulerNode.
   * @param schedulerKey SchedulerRequestKey.
   * @param rmContainer RMContainer.
   * @return ContainerId.
   */
public ContainerId matchContainerToOutstandingIncreaseReq(SchedulerNode node, SchedulerRequestKey schedulerKey, RMContainer rmContainer) {
    ContainerId retVal = null;
    Container container = rmContainer.getContainer();
    Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap = outstandingIncreases.get(schedulerKey);
    if (resourceMap != null) {
        Map<NodeId, Set<ContainerId>> locationMap = resourceMap.get(container.getResource());
        if (locationMap != null) {
            Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
            if (containerIds != null && !containerIds.isEmpty()) {
                retVal = containerIds.iterator().next();
            }
        }
    }
    // We also need to add these requests back.. to be reallocated.
    if (resourceMap != null && retVal == null) {
        Map<SchedulerRequestKey, Map<String, ResourceRequest>> reqsToUpdate = new HashMap<>();
        Map<String, ResourceRequest> resMap = createResourceRequests(rmContainer, node, schedulerKey, rmContainer.getContainer().getResource());
        reqsToUpdate.put(schedulerKey, resMap);
        appSchedulingInfo.addToPlacementSets(true, reqsToUpdate);
        return UNDEFINED;
    }
    return retVal;
}

Example 4 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class ContainerUpdateContext method checkAndAddToOutstandingIncreases.

/**
   * Add the container to outstanding increases.
   * @param rmContainer RMContainer.
   * @param schedulerNode SchedulerNode.
   * @param updateRequest UpdateContainerRequest.
   * @return true if updated to outstanding increases was successful.
   */
public synchronized boolean checkAndAddToOutstandingIncreases(RMContainer rmContainer, SchedulerNode schedulerNode, UpdateContainerRequest updateRequest) {
    Container container = rmContainer.getContainer();
    SchedulerRequestKey schedulerKey = SchedulerRequestKey.create(updateRequest, rmContainer.getAllocatedSchedulerKey());
    Map<Resource, Map<NodeId, Set<ContainerId>>> resourceMap = outstandingIncreases.get(schedulerKey);
    if (resourceMap == null) {
        resourceMap = new HashMap<>();
        outstandingIncreases.put(schedulerKey, resourceMap);
    } else {
        // Updating Resource for and existing increase container
        if (ContainerUpdateType.INCREASE_RESOURCE == updateRequest.getContainerUpdateType()) {
            cancelPreviousRequest(schedulerNode, schedulerKey);
        } else {
            return false;
        }
    }
    Resource resToIncrease = getResourceToIncrease(updateRequest, rmContainer);
    Map<NodeId, Set<ContainerId>> locationMap = resourceMap.get(resToIncrease);
    if (locationMap == null) {
        locationMap = new HashMap<>();
        resourceMap.put(resToIncrease, locationMap);
    }
    Set<ContainerId> containerIds = locationMap.get(container.getNodeId());
    if (containerIds == null) {
        containerIds = new HashSet<>();
        locationMap.put(container.getNodeId(), containerIds);
    }
    if (outstandingDecreases.containsKey(container.getId())) {
        return false;
    }
    containerIds.add(container.getId());
    if (!Resources.isNone(resToIncrease)) {
        Map<SchedulerRequestKey, Map<String, ResourceRequest>> updateResReqs = new HashMap<>();
        Map<String, ResourceRequest> resMap = createResourceRequests(rmContainer, schedulerNode, schedulerKey, resToIncrease);
        updateResReqs.put(schedulerKey, resMap);
        appSchedulingInfo.addToPlacementSets(false, updateResReqs);
    }
    return true;
}

Example 5 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class SchedulerApplicationAttempt method showRequests.

public void showRequests() {
    if (LOG.isDebugEnabled()) {
        try {
            readLock.lock();
            for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) {
                SchedulingPlacementSet ps = getSchedulingPlacementSet(schedulerKey);
                if (ps != null && ps.getOutstandingAsksCount(ResourceRequest.ANY) > 0) {
                    LOG.debug("showRequests:" + " application=" + getApplicationId() + " headRoom=" + getHeadroom() + " currentConsumption=" + attemptResourceUsage.getUsed().getMemorySize());
                    ps.showRequests();
                }
            }
        } finally {
            readLock.unlock();
        }
    }
}