Examples with ResourceUsage org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage used on opensource projects

Example 1 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class CSQueueUtils method updateQueueStatistics.

/**
   * <p>
   * Update Queue Statistics:
   * </p>
   *  
   * <li>used-capacity/absolute-used-capacity by partition</li> 
   * <li>non-partitioned max-avail-resource to queue</li>
   * 
   * <p>
   * When nodePartition is null, all partition of
   * used-capacity/absolute-used-capacity will be updated.
   * </p>
   */
@Lock(CSQueue.class)
public static void updateQueueStatistics(final ResourceCalculator rc, final Resource cluster, final Resource minimumAllocation, final CSQueue childQueue, final RMNodeLabelsManager nlm, final String nodePartition) {
    QueueCapacities queueCapacities = childQueue.getQueueCapacities();
    ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
    if (nodePartition == null) {
        for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(), queueResourceUsage.getNodePartitionsSet())) {
            updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster), minimumAllocation, queueResourceUsage, queueCapacities, partition);
        }
    } else {
        updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster), minimumAllocation, queueResourceUsage, queueCapacities, nodePartition);
    }
    // Update queue metrics w.r.t node labels. In a generic way, we can
    // calculate available resource from all labels in cluster.
    childQueue.getMetrics().setAvailableResourcesToQueue(getMaxAvailableResourceToQueue(rc, nlm, childQueue, cluster));
}

Example 2 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class UsersManager method updateNonActiveUsersResourceUsage.

private void updateNonActiveUsersResourceUsage(String userName) {
    try {
        this.writeLock.lock();
        // For UT case: We might need to add the user to users list.
        User user = getUser(userName);
        if (user == null)
            return;
        ResourceUsage resourceUsage = user.getResourceUsage();
        // non-active to active list.
        if (activeUsersSet.contains(userName)) {
            activeUsersSet.remove(userName);
            nonActiveUsersSet.add(userName);
            // moved from active to non-active.
            for (String partition : resourceUsage.getNodePartitionsSet()) {
                totalResUsageForActiveUsers.decUsed(partition, resourceUsage.getUsed(partition));
                totalResUsageForNonActiveUsers.incUsed(partition, resourceUsage.getUsed(partition));
                if (LOG.isDebugEnabled()) {
                    LOG.debug("User '" + userName + "' has become non-active.Hence move user to non-active list." + "Active users size = " + activeUsersSet.size() + "Non-active users size = " + nonActiveUsersSet.size() + "Total Resource usage for active users=" + totalResUsageForActiveUsers.getAllUsed() + "." + "Total Resource usage for non-active users=" + totalResUsageForNonActiveUsers.getAllUsed());
                }
            }
        }
    } finally {
        this.writeLock.unlock();
    }
}

Example 3 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class LeafQueue method recalculateQueueUsageRatio.

/**
   * Recalculate QueueUsage Ratio.
   *
   * @param clusterResource
   *          Total Cluster Resource
   * @param nodePartition
   *          Partition
   */
public void recalculateQueueUsageRatio(Resource clusterResource, String nodePartition) {
    try {
        writeLock.lock();
        ResourceUsage queueResourceUsage = getQueueResourceUsage();
        if (nodePartition == null) {
            for (String partition : Sets.union(getQueueCapacities().getNodePartitionsSet(), queueResourceUsage.getNodePartitionsSet())) {
                usersManager.updateUsageRatio(partition, clusterResource);
            }
        } else {
            usersManager.updateUsageRatio(nodePartition, clusterResource);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 4 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method checkPendingResource.

public void checkPendingResource(CSQueue queue, String partition, int pending) {
    ResourceUsage ru = queue.getQueueResourceUsage();
    Assert.assertEquals(pending, ru.getPending(partition).getMemorySize());
}

Example 5 with ResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method mockLeafQueue.

@SuppressWarnings("rawtypes")
LeafQueue mockLeafQueue(ParentQueue p, Resource tot, int i, Resource[] abs, Resource[] used, Resource[] pending, Resource[] reserved, int[] apps, Resource[] gran) {
    LeafQueue lq = mock(LeafQueue.class);
    ResourceCalculator rc = mCS.getResourceCalculator();
    List<ApplicationAttemptId> appAttemptIdList = new ArrayList<ApplicationAttemptId>();
    when(lq.getTotalPendingResourcesConsideringUserLimit(isA(Resource.class), isA(String.class), eq(false))).thenReturn(pending[i]);
    when(lq.getTotalPendingResourcesConsideringUserLimit(isA(Resource.class), isA(String.class), eq(true))).thenReturn(Resources.componentwiseMax(Resources.subtract(pending[i], reserved[i] == null ? Resources.none() : reserved[i]), Resources.none()));
    // need to set pending resource in resource usage as well
    ResourceUsage ru = new ResourceUsage();
    ru.setPending(pending[i]);
    ru.setUsed(used[i]);
    ru.setReserved(reserved[i]);
    when(lq.getQueueResourceUsage()).thenReturn(ru);
    // consider moving where CapacityScheduler::comparator accessible
    final NavigableSet<FiCaSchedulerApp> qApps = new TreeSet<FiCaSchedulerApp>(new Comparator<FiCaSchedulerApp>() {

        @Override
        public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) {
            return a1.getApplicationAttemptId().compareTo(a2.getApplicationAttemptId());
        }
    });
    // applications are added in global L->R order in queues
    if (apps[i] != 0) {
        Resource aUsed = Resources.divideAndCeil(rc, used[i], apps[i]);
        Resource aPending = Resources.divideAndCeil(rc, pending[i], apps[i]);
        Resource aReserve = Resources.divideAndCeil(rc, reserved[i], apps[i]);
        for (int a = 0; a < apps[i]; ++a) {
            FiCaSchedulerApp mockFiCaApp = mockApp(i, appAlloc, aUsed, aPending, aReserve, gran[i]);
            qApps.add(mockFiCaApp);
            ++appAlloc;
            appAttemptIdList.add(mockFiCaApp.getApplicationAttemptId());
        }
        when(mCS.getAppsInQueue("queue" + (char) ('A' + i - 1))).thenReturn(appAttemptIdList);
    }
    when(lq.getApplications()).thenReturn(qApps);
    @SuppressWarnings("unchecked") OrderingPolicy<FiCaSchedulerApp> so = mock(OrderingPolicy.class);
    when(so.getPreemptionIterator()).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            return qApps.descendingIterator();
        }
    });
    when(lq.getOrderingPolicy()).thenReturn(so);
    if (setAMResourcePercent != 0.0f) {
        when(lq.getMaxAMResourcePerQueuePercent()).thenReturn(setAMResourcePercent);
    }
    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
    when(lq.getReadLock()).thenReturn(lock.readLock());
    when(lq.getPriority()).thenReturn(Priority.newInstance(0));
    p.getChildQueues().add(lq);
    return lq;
}