Examples with Lock org.apache.hadoop.yarn.server.utils.Lock used on opensource projects

Example 1 with Lock

use of org.apache.hadoop.yarn.server.utils.Lock in project hadoop by apache.

the class CSQueueUtils method updateQueueStatistics.

/**
   * <p>
   * Update Queue Statistics:
   * </p>
   *  
   * <li>used-capacity/absolute-used-capacity by partition</li> 
   * <li>non-partitioned max-avail-resource to queue</li>
   * 
   * <p>
   * When nodePartition is null, all partition of
   * used-capacity/absolute-used-capacity will be updated.
   * </p>
   */
@Lock(CSQueue.class)
public static void updateQueueStatistics(final ResourceCalculator rc, final Resource cluster, final Resource minimumAllocation, final CSQueue childQueue, final RMNodeLabelsManager nlm, final String nodePartition) {
    QueueCapacities queueCapacities = childQueue.getQueueCapacities();
    ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
    if (nodePartition == null) {
        for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(), queueResourceUsage.getNodePartitionsSet())) {
            updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster), minimumAllocation, queueResourceUsage, queueCapacities, partition);
        }
    } else {
        updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster), minimumAllocation, queueResourceUsage, queueCapacities, nodePartition);
    }
    // Update queue metrics w.r.t node labels. In a generic way, we can
    // calculate available resource from all labels in cluster.
    childQueue.getMetrics().setAvailableResourcesToQueue(getMaxAvailableResourceToQueue(rc, nlm, childQueue, cluster));
}

Example 2 with Lock

use of org.apache.hadoop.yarn.server.utils.Lock in project hadoop by apache.

the class CapacityScheduler method allocate.

@Override
@Lock(Lock.NoLock.class)
public Allocation allocate(ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests) {
    FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId);
    if (application == null) {
        return EMPTY_ALLOCATION;
    }
    // Handle all container updates
    handleContainerUpdates(application, updateRequests);
    // Release containers
    releaseContainers(release, application);
    LeafQueue updateDemandForQueue = null;
    // Sanity check for new allocation requests
    normalizeRequests(ask);
    Allocation allocation;
    // when the allocate comes in
    try {
        application.getWriteLock().lock();
        if (application.isStopped()) {
            return EMPTY_ALLOCATION;
        }
        // Process resource requests
        if (!ask.isEmpty()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("allocate: pre-update " + applicationAttemptId + " ask size =" + ask.size());
                application.showRequests();
            }
            // Update application requests
            if (application.updateResourceRequests(ask)) {
                updateDemandForQueue = (LeafQueue) application.getQueue();
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("allocate: post-update");
                application.showRequests();
            }
        }
        application.updateBlacklist(blacklistAdditions, blacklistRemovals);
        allocation = application.getAllocation(getResourceCalculator(), getClusterResource(), getMinimumResourceCapability());
    } finally {
        application.getWriteLock().unlock();
    }
    if (updateDemandForQueue != null && !application.isWaitingForAMContainer()) {
        updateDemandForQueue.getOrderingPolicy().demandUpdated(application);
    }
    return allocation;
}

Example 3 with Lock

use of org.apache.hadoop.yarn.server.utils.Lock in project hadoop by apache.

the class FifoScheduler method completedContainerInternal.

@Lock(FifoScheduler.class)
@Override
protected synchronized void completedContainerInternal(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) {
    // Get the application for the finished container
    Container container = rmContainer.getContainer();
    FifoAppAttempt application = getCurrentAttemptForContainer(container.getId());
    ApplicationId appId = container.getId().getApplicationAttemptId().getApplicationId();
    // Get the node on which the container was allocated
    FiCaSchedulerNode node = (FiCaSchedulerNode) getNode(container.getNodeId());
    if (application == null) {
        LOG.info("Unknown application: " + appId + " released container " + container.getId() + " on node: " + node + " with event: " + event);
        return;
    }
    // Inform the application
    application.containerCompleted(rmContainer, containerStatus, event, RMNodeLabelsManager.NO_LABEL);
    // Inform the node
    node.releaseContainer(rmContainer.getContainerId(), false);
    // Update total usage
    Resources.subtractFrom(usedResource, container.getResource());
    LOG.info("Application attempt " + application.getApplicationAttemptId() + " released container " + container.getId() + " on node: " + node + " with event: " + event);
}

Example 4 with Lock

use of org.apache.hadoop.yarn.server.utils.Lock in project hadoop by apache.

the class LeafQueue method computeUserLimitAndSetHeadroom.

// It doesn't necessarily to hold application's lock here.
@Lock({ LeafQueue.class })
Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application, Resource clusterResource, String nodePartition, SchedulingMode schedulingMode) {
    String user = application.getUser();
    User queueUser = getUser(user);
    // Compute user limit respect requested labels,
    // TODO, need consider headroom respect labels also
    Resource userLimit = getResourceLimitForActiveUsers(application.getUser(), clusterResource, nodePartition, schedulingMode);
    setQueueResourceLimitsInfo(clusterResource);
    Resource headroom = getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, userLimit, nodePartition);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Headroom calculation for user " + user + ": " + " userLimit=" + userLimit + " queueMaxAvailRes=" + cachedResourceLimitsForHeadroom.getLimit() + " consumed=" + queueUser.getUsed() + " headroom=" + headroom + " partition=" + nodePartition);
    }
    CapacityHeadroomProvider headroomProvider = new CapacityHeadroomProvider(queueUser, this, application, queueResourceLimitsInfo);
    application.setHeadroomProvider(headroomProvider);
    metrics.setAvailableResourcesToUser(user, headroom);
    return userLimit;
}