Examples with ApplicationResourceUsageReport org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport used on opensource projects

Example 1 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestTypeConverter method testFromYarn.

@Test
public void testFromYarn() throws Exception {
    int appStartTime = 612354;
    int appFinishTime = 612355;
    YarnApplicationState state = YarnApplicationState.RUNNING;
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationReport applicationReport = Records.newRecord(ApplicationReport.class);
    applicationReport.setApplicationId(applicationId);
    applicationReport.setYarnApplicationState(state);
    applicationReport.setStartTime(appStartTime);
    applicationReport.setFinishTime(appFinishTime);
    applicationReport.setUser("TestTypeConverter-user");
    applicationReport.setPriority(Priority.newInstance(3));
    ApplicationResourceUsageReport appUsageRpt = Records.newRecord(ApplicationResourceUsageReport.class);
    Resource r = Records.newRecord(Resource.class);
    r.setMemorySize(2048);
    appUsageRpt.setNeededResources(r);
    appUsageRpt.setNumReservedContainers(1);
    appUsageRpt.setNumUsedContainers(3);
    appUsageRpt.setReservedResources(r);
    appUsageRpt.setUsedResources(r);
    applicationReport.setApplicationResourceUsageReport(appUsageRpt);
    JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
    Assert.assertEquals(appStartTime, jobStatus.getStartTime());
    Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());
    Assert.assertEquals(state.toString(), jobStatus.getState().toString());
    Assert.assertEquals(JobPriority.NORMAL, jobStatus.getPriority());
}

Example 2 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestClientRMService method getRMApp.

private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, ApplicationId applicationId3, YarnConfiguration config, String queueName, final long memorySeconds, final long vcoreSeconds, String appNodeLabelExpression, String amNodeLabelExpression) {
    ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class);
    when(asContext.getMaxAppAttempts()).thenReturn(1);
    when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
    when(asContext.getPriority()).thenReturn(Priority.newInstance(0));
    RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, System.currentTimeMillis(), "YARN", null, BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, Resource.newInstance(1024, 1), 1)) {

        @Override
        public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
            ApplicationReport report = super.createAndGetApplicationReport(clientUserName, allowAccess);
            ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport();
            usageReport.setMemorySeconds(memorySeconds);
            usageReport.setVcoreSeconds(vcoreSeconds);
            report.setApplicationResourceUsageReport(usageReport);
            return report;
        }
    });
    app.getAMResourceRequest().setNodeLabelExpression(amNodeLabelExpression);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(123456, 1), 1);
    RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, null, app));
    Container container = Container.newInstance(ContainerId.newContainerId(attemptId, 1), null, "", null, null, null);
    RMContainerImpl containerimpl = spy(new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), attemptId, null, "", rmContext));
    Map<ApplicationAttemptId, RMAppAttempt> attempts = new HashMap<ApplicationAttemptId, RMAppAttempt>();
    attempts.put(attemptId, rmAppAttemptImpl);
    when(app.getCurrentAppAttempt()).thenReturn(rmAppAttemptImpl);
    when(app.getAppAttempts()).thenReturn(attempts);
    when(app.getApplicationPriority()).thenReturn(Priority.newInstance(0));
    when(rmAppAttemptImpl.getMasterContainer()).thenReturn(container);
    ResourceScheduler rs = mock(ResourceScheduler.class);
    when(rmContext.getScheduler()).thenReturn(rs);
    when(rmContext.getScheduler().getRMContainer(any(ContainerId.class))).thenReturn(containerimpl);
    SchedulerAppReport sAppReport = mock(SchedulerAppReport.class);
    when(rmContext.getScheduler().getSchedulerAppInfo(any(ApplicationAttemptId.class))).thenReturn(sAppReport);
    List<RMContainer> rmContainers = new ArrayList<RMContainer>();
    rmContainers.add(containerimpl);
    when(rmContext.getScheduler().getSchedulerAppInfo(attemptId).getLiveContainers()).thenReturn(rmContainers);
    ContainerStatus cs = mock(ContainerStatus.class);
    when(containerimpl.completed()).thenReturn(false);
    when(containerimpl.getDiagnosticsInfo()).thenReturn("N/A");
    when(containerimpl.getContainerExitStatus()).thenReturn(0);
    when(containerimpl.getContainerState()).thenReturn(ContainerState.COMPLETE);
    return app;
}

Example 3 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestClientRMService method testGetApplicationReport.

@Test
public void testGetApplicationReport() throws Exception {
    YarnScheduler yarnScheduler = mock(YarnScheduler.class);
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    ApplicationId appId1 = getApplicationId(1);
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, null, mockAclsManager, null, null);
    try {
        RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
        GetApplicationReportRequest request = recordFactory.newRecordInstance(GetApplicationReportRequest.class);
        request.setApplicationId(appId1);
        GetApplicationReportResponse response = rmService.getApplicationReport(request);
        ApplicationReport report = response.getApplicationReport();
        ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport();
        Assert.assertEquals(10, usageReport.getMemorySeconds());
        Assert.assertEquals(3, usageReport.getVcoreSeconds());
        Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
        Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
        // if application has am node label set to blank
        ApplicationId appId2 = getApplicationId(2);
        when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId2)).thenReturn(true);
        request.setApplicationId(appId2);
        response = rmService.getApplicationReport(request);
        report = response.getApplicationReport();
        Assert.assertEquals(NodeLabel.DEFAULT_NODE_LABEL_PARTITION, report.getAmNodeLabelExpression());
        Assert.assertEquals(NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET, report.getAppNodeLabelExpression());
        // if application has am node label set to blank
        ApplicationId appId3 = getApplicationId(3);
        when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId3)).thenReturn(true);
        request.setApplicationId(appId3);
        response = rmService.getApplicationReport(request);
        report = response.getApplicationReport();
        Assert.assertEquals("high-mem", report.getAmNodeLabelExpression());
        Assert.assertEquals("high-mem", report.getAppNodeLabelExpression());
        // if application id is null
        GetApplicationReportRequest invalidRequest = recordFactory.newRecordInstance(GetApplicationReportRequest.class);
        invalidRequest.setApplicationId(null);
        try {
            rmService.getApplicationReport(invalidRequest);
        } catch (YarnException e) {
            // rmService should return a ApplicationNotFoundException
            // when a null application id is provided
            Assert.assertTrue(e instanceof ApplicationNotFoundException);
        }
    } finally {
        rmService.close();
    }
}

Example 4 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class BuilderUtils method newApplicationResourceUsageReport.

public static ApplicationResourceUsageReport newApplicationResourceUsageReport(int numUsedContainers, int numReservedContainers, Resource usedResources, Resource reservedResources, Resource neededResources, long memorySeconds, long vcoreSeconds, long preemptedMemorySeconds, long preemptedVcoreSeconds) {
    ApplicationResourceUsageReport report = recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
    report.setNumUsedContainers(numUsedContainers);
    report.setNumReservedContainers(numReservedContainers);
    report.setUsedResources(usedResources);
    report.setReservedResources(reservedResources);
    report.setNeededResources(neededResources);
    report.setMemorySeconds(memorySeconds);
    report.setVcoreSeconds(vcoreSeconds);
    report.setPreemptedMemorySeconds(preemptedMemorySeconds);
    report.setPreemptedVcoreSeconds(preemptedVcoreSeconds);
    return report;
}

Example 5 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class RMAppAttemptImpl method getApplicationResourceUsageReport.

@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
    this.readLock.lock();
    try {
        ApplicationResourceUsageReport report = scheduler.getAppResourceUsageReport(this.getAppAttemptId());
        if (report == null) {
            report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
        }
        AggregateAppResourceUsage resUsage = this.attemptMetrics.getAggregateAppResourceUsage();
        report.setMemorySeconds(resUsage.getMemorySeconds());
        report.setVcoreSeconds(resUsage.getVcoreSeconds());
        report.setPreemptedMemorySeconds(this.attemptMetrics.getPreemptedMemory());
        report.setPreemptedVcoreSeconds(this.attemptMetrics.getPreemptedVcore());
        return report;
    } finally {
        this.readLock.unlock();
    }
}