Examples with AggregateAppResourceUsage org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage used on opensource projects

Example 1 with AggregateAppResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage in project hadoop by apache.

the class TestContainerResourceUsage method testUsageWithOneAttemptAndOneContainer.

@Test(timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
    MockRM rm = new MockRM(conf);
    rm.start();
    MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
    nm.registerNode();
    RMApp app0 = rm.submitApp(200);
    RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
    Assert.assertTrue("Before app submittion, memory seconds should have been 0 but was " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() == 0);
    Assert.assertTrue("Before app submission, vcore seconds should have been 0 but was " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() == 0);
    RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
    nm.nodeHeartbeat(true);
    MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
    am0.registerAppAttempt();
    RMContainer rmContainer = rm.getResourceScheduler().getRMContainer(attempt0.getMasterContainer().getId());
    // Allow metrics to accumulate.
    int sleepInterval = 1000;
    int cumulativeSleepTime = 0;
    while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
        Thread.sleep(sleepInterval);
        cumulativeSleepTime += sleepInterval;
    }
    rmAppMetrics = app0.getRMAppMetrics();
    Assert.assertTrue("While app is running, memory seconds should be >0 but is " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() > 0);
    Assert.assertTrue("While app is running, vcore seconds should be >0 but is " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() > 0);
    MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);
    AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
    rmAppMetrics = app0.getRMAppMetrics();
    Assert.assertEquals("Unexpected MemorySeconds value", ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
    Assert.assertEquals("Unexpected VcoreSeconds value", ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
    rm.stop();
}

Example 2 with AggregateAppResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage in project hadoop by apache.

the class TestContainerResourceUsage method calculateContainerResourceMetrics.

private AggregateAppResourceUsage calculateContainerResourceMetrics(RMContainer rmContainer) {
    Resource resource = rmContainer.getContainer().getResource();
    long usedMillis = rmContainer.getFinishTime() - rmContainer.getCreationTime();
    long memorySeconds = resource.getMemorySize() * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = resource.getVirtualCores() * usedMillis / DateUtils.MILLIS_PER_SECOND;
    return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
}

Example 3 with AggregateAppResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage in project hadoop by apache.

the class SchedulerApplicationAttempt method getResourceUsageReport.

public ApplicationResourceUsageReport getResourceUsageReport() {
    try {
        writeLock.lock();
        AggregateAppResourceUsage runningResourceUsage = getRunningAggregateAppResourceUsage();
        Resource usedResourceClone = Resources.clone(attemptResourceUsage.getAllUsed());
        Resource reservedResourceClone = Resources.clone(attemptResourceUsage.getReserved());
        Resource cluster = rmContext.getScheduler().getClusterResource();
        ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
        float queueUsagePerc = 0.0f;
        float clusterUsagePerc = 0.0f;
        if (!calc.isInvalidDivisor(cluster)) {
            float queueCapacityPerc = queue.getQueueInfo(false, false).getCapacity();
            if (queueCapacityPerc != 0) {
                queueUsagePerc = calc.divide(cluster, usedResourceClone, Resources.multiply(cluster, queueCapacityPerc)) * 100;
            }
            clusterUsagePerc = calc.divide(cluster, usedResourceClone, cluster) * 100;
        }
        return ApplicationResourceUsageReport.newInstance(liveContainers.size(), reservedContainers.size(), usedResourceClone, reservedResourceClone, Resources.add(usedResourceClone, reservedResourceClone), runningResourceUsage.getMemorySeconds(), runningResourceUsage.getVcoreSeconds(), queueUsagePerc, clusterUsagePerc, 0, 0);
    } finally {
        writeLock.unlock();
    }
}

Example 4 with AggregateAppResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage in project hadoop by apache.

the class SchedulerApplicationAttempt method getRunningAggregateAppResourceUsage.

private AggregateAppResourceUsage getRunningAggregateAppResourceUsage() {
    long currentTimeMillis = System.currentTimeMillis();
    // recently.
    if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
        long memorySeconds = 0;
        long vcoreSeconds = 0;
        for (RMContainer rmContainer : this.liveContainers.values()) {
            long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
            Resource resource = rmContainer.getContainer().getResource();
            memorySeconds += resource.getMemorySize() * usedMillis / DateUtils.MILLIS_PER_SECOND;
            vcoreSeconds += resource.getVirtualCores() * usedMillis / DateUtils.MILLIS_PER_SECOND;
        }
        lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
        lastMemorySeconds = memorySeconds;
        lastVcoreSeconds = vcoreSeconds;
    }
    return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds);
}

Example 5 with AggregateAppResourceUsage

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage in project hadoop by apache.

the class RMAppImpl method getRMAppMetrics.

@Override
public RMAppMetrics getRMAppMetrics() {
    Resource resourcePreempted = Resource.newInstance(0, 0);
    int numAMContainerPreempted = 0;
    int numNonAMContainerPreempted = 0;
    long memorySeconds = 0;
    long vcoreSeconds = 0;
    long preemptedMemorySeconds = 0;
    long preemptedVcoreSeconds = 0;
    for (RMAppAttempt attempt : attempts.values()) {
        if (null != attempt) {
            RMAppAttemptMetrics attemptMetrics = attempt.getRMAppAttemptMetrics();
            Resources.addTo(resourcePreempted, attemptMetrics.getResourcePreempted());
            numAMContainerPreempted += attemptMetrics.getIsPreempted() ? 1 : 0;
            numNonAMContainerPreempted += attemptMetrics.getNumNonAMContainersPreempted();
            // getAggregateAppResourceUsage() will calculate resource usage stats
            // for both running and finished containers.
            AggregateAppResourceUsage resUsage = attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
            memorySeconds += resUsage.getMemorySeconds();
            vcoreSeconds += resUsage.getVcoreSeconds();
            preemptedMemorySeconds += attemptMetrics.getPreemptedMemory();
            preemptedVcoreSeconds += attemptMetrics.getPreemptedVcore();
        }
    }
    return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted, numAMContainerPreempted, memorySeconds, vcoreSeconds, preemptedMemorySeconds, preemptedVcoreSeconds);
}