Examples with ApplicationResourceUsageReport org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport used on opensource projects

Example 11 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestClientRMService method testGetApplicationResourceUsageReportDummy.

@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException, IOException {
    ApplicationAttemptId attemptId = getApplicationAttemptId(1);
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class);
    YarnConfiguration config = new YarnConfiguration();
    RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, null, null);
    ApplicationResourceUsageReport report = rmAppAttemptImpl.getApplicationResourceUsageReport();
    assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}

Example 12 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class FiCaSchedulerApp method getResourceUsageReport.

/**
   * Recalculates the per-app, percent of queue metric, specific to the
   * Capacity Scheduler.
   */
@Override
public ApplicationResourceUsageReport getResourceUsageReport() {
    try {
        // Use write lock here because
        // SchedulerApplicationAttempt#getResourceUsageReport updated fields
        // TODO: improve this
        writeLock.lock();
        ApplicationResourceUsageReport report = super.getResourceUsageReport();
        Resource cluster = rmContext.getScheduler().getClusterResource();
        Resource totalPartitionRes = rmContext.getNodeLabelManager().getResourceByLabel(getAppAMNodePartitionName(), cluster);
        ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
        if (!calc.isInvalidDivisor(totalPartitionRes)) {
            float queueAbsMaxCapPerPartition = ((AbstractCSQueue) getQueue()).getQueueCapacities().getAbsoluteCapacity(getAppAMNodePartitionName());
            float queueUsagePerc = calc.divide(totalPartitionRes, report.getUsedResources(), Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition)) * 100;
            report.setQueueUsagePercentage(queueUsagePerc);
        }
        return report;
    } finally {
        writeLock.unlock();
    }
}

Example 13 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestApplicationACLs method verifyEnemyAppReport.

private void verifyEnemyAppReport(ApplicationReport appReport) {
    Assert.assertEquals("Enemy should not see app host!", UNAVAILABLE, appReport.getHost());
    Assert.assertEquals("Enemy should not see app rpc port!", -1, appReport.getRpcPort());
    Assert.assertEquals("Enemy should not see app client token!", null, appReport.getClientToAMToken());
    Assert.assertEquals("Enemy should not see app diagnostics!", UNAVAILABLE, appReport.getDiagnostics());
    Assert.assertEquals("Enemy should not see app tracking url!", UNAVAILABLE, appReport.getTrackingUrl());
    Assert.assertEquals("Enemy should not see app original tracking url!", UNAVAILABLE, appReport.getOriginalTrackingUrl());
    ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport();
    Assert.assertEquals("Enemy should not see app used containers", -1, usageReport.getNumUsedContainers());
    Assert.assertEquals("Enemy should not see app reserved containers", -1, usageReport.getNumReservedContainers());
    Assert.assertEquals("Enemy should not see app used resources", -1, usageReport.getUsedResources().getMemorySize());
    Assert.assertEquals("Enemy should not see app reserved resources", -1, usageReport.getReservedResources().getMemorySize());
    Assert.assertEquals("Enemy should not see app needed resources", -1, usageReport.getNeededResources().getMemorySize());
}

Example 14 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestCapacitySchedulerNodeLabelUpdate method testRequestContainerAfterNodePartitionUpdated.

@Test(timeout = 60000)
public void testRequestContainerAfterNodePartitionUpdated() throws Exception {
    // set node -> label
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z"));
    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
    // inject node label manager
    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 2048);
    MockNM nm2 = rm.registerNode("h2:1234", 2048);
    MockNM nm3 = rm.registerNode("h3:1234", 2048);
    ContainerId containerId;
    // launch an app to queue a1 (label = x), and check all container will
    // be allocated in h1
    RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
    ApplicationResourceUsageReport appResourceUsageReport = rm.getResourceScheduler().getAppResourceUsageReport(am1.getApplicationAttemptId());
    Assert.assertEquals(1024, appResourceUsageReport.getUsedResources().getMemorySize());
    Assert.assertEquals(1, appResourceUsageReport.getUsedResources().getVirtualCores());
    // request a container.
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED);
    appResourceUsageReport = rm.getResourceScheduler().getAppResourceUsageReport(am1.getApplicationAttemptId());
    Assert.assertEquals(2048, appResourceUsageReport.getUsedResources().getMemorySize());
    Assert.assertEquals(2, appResourceUsageReport.getUsedResources().getVirtualCores());
    LeafQueue queue = (LeafQueue) ((CapacityScheduler) rm.getResourceScheduler()).getQueue("a");
    ArrayList<UserInfo> users = queue.getUsersManager().getUsersInfo();
    for (UserInfo userInfo : users) {
        if (userInfo.getUsername().equals("user")) {
            ResourceInfo resourcesUsed = userInfo.getResourcesUsed();
            Assert.assertEquals(2048, resourcesUsed.getMemorySize());
            Assert.assertEquals(2, resourcesUsed.getvCores());
        }
    }
    rm.stop();
}

Example 15 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TypeConverter method fromYarn.

public static JobStatus fromYarn(ApplicationReport application, String jobFile) {
    String trackingUrl = application.getTrackingUrl();
    trackingUrl = trackingUrl == null ? "" : trackingUrl;
    JobStatus jobStatus = new JobStatus(TypeConverter.fromYarn(application.getApplicationId()), 0.0f, 0.0f, 0.0f, 0.0f, TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()), fromYarnApplicationPriority((application.getPriority() == null) ? 0 : application.getPriority().getPriority()), application.getUser(), application.getName(), application.getQueue(), jobFile, trackingUrl, false);
    // Set AM tracking url
    jobStatus.setSchedulingInfo(trackingUrl);
    jobStatus.setStartTime(application.getStartTime());
    jobStatus.setFinishTime(application.getFinishTime());
    jobStatus.setFailureInfo(application.getDiagnostics());
    ApplicationResourceUsageReport resourceUsageReport = application.getApplicationResourceUsageReport();
    if (resourceUsageReport != null) {
        jobStatus.setNeededMem((int) resourceUsageReport.getNeededResources().getMemorySize());
        jobStatus.setNumReservedSlots(resourceUsageReport.getNumReservedContainers());
        jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
        jobStatus.setReservedMem((int) resourceUsageReport.getReservedResources().getMemorySize());
        jobStatus.setUsedMem((int) resourceUsageReport.getUsedResources().getMemorySize());
    }
    return jobStatus;
}