Examples with ApplicationResourceUsageReport org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport used on opensource projects

Example 16 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestTypeConverter method testFromYarnApplicationReport.

@Test
public void testFromYarnApplicationReport() {
    ApplicationId mockAppId = mock(ApplicationId.class);
    when(mockAppId.getClusterTimestamp()).thenReturn(12345L);
    when(mockAppId.getId()).thenReturn(6789);
    ApplicationReport mockReport = mock(ApplicationReport.class);
    when(mockReport.getTrackingUrl()).thenReturn("dummy-tracking-url");
    when(mockReport.getApplicationId()).thenReturn(mockAppId);
    when(mockReport.getYarnApplicationState()).thenReturn(YarnApplicationState.KILLED);
    when(mockReport.getUser()).thenReturn("dummy-user");
    when(mockReport.getQueue()).thenReturn("dummy-queue");
    when(mockReport.getPriority()).thenReturn(Priority.newInstance(4));
    String jobFile = "dummy-path/job.xml";
    try {
        JobStatus status = TypeConverter.fromYarn(mockReport, jobFile);
    } catch (NullPointerException npe) {
        Assert.fail("Type converstion from YARN fails for jobs without " + "ApplicationUsageReport");
    }
    ApplicationResourceUsageReport appUsageRpt = Records.newRecord(ApplicationResourceUsageReport.class);
    Resource r = Records.newRecord(Resource.class);
    r.setMemorySize(2048);
    appUsageRpt.setNeededResources(r);
    appUsageRpt.setNumReservedContainers(1);
    appUsageRpt.setNumUsedContainers(3);
    appUsageRpt.setReservedResources(r);
    appUsageRpt.setUsedResources(r);
    when(mockReport.getApplicationResourceUsageReport()).thenReturn(appUsageRpt);
    JobStatus status = TypeConverter.fromYarn(mockReport, jobFile);
    Assert.assertNotNull("fromYarn returned null status", status);
    Assert.assertEquals("jobFile set incorrectly", "dummy-path/job.xml", status.getJobFile());
    Assert.assertEquals("queue set incorrectly", "dummy-queue", status.getQueue());
    Assert.assertEquals("trackingUrl set incorrectly", "dummy-tracking-url", status.getTrackingUrl());
    Assert.assertEquals("user set incorrectly", "dummy-user", status.getUsername());
    Assert.assertEquals("schedulingInfo set incorrectly", "dummy-tracking-url", status.getSchedulingInfo());
    Assert.assertEquals("jobId set incorrectly", 6789, status.getJobID().getId());
    Assert.assertEquals("state set incorrectly", JobStatus.State.KILLED, status.getState());
    Assert.assertEquals("needed mem info set incorrectly", 2048, status.getNeededMem());
    Assert.assertEquals("num rsvd slots info set incorrectly", 1, status.getNumReservedSlots());
    Assert.assertEquals("num used slots info set incorrectly", 3, status.getNumUsedSlots());
    Assert.assertEquals("rsvd mem info set incorrectly", 2048, status.getReservedMem());
    Assert.assertEquals("used mem info set incorrectly", 2048, status.getUsedMem());
    Assert.assertEquals("priority set incorrectly", JobPriority.HIGH, status.getPriority());
}

Example 17 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestResourceMgrDelegate method getApplicationReport.

private ApplicationReport getApplicationReport(YarnApplicationState yarnApplicationState, FinalApplicationStatus finalApplicationStatus) {
    ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
    ApplicationResourceUsageReport appResources = Mockito.mock(ApplicationResourceUsageReport.class);
    Mockito.when(appReport.getApplicationId()).thenReturn(ApplicationId.newInstance(0, 0));
    Mockito.when(appResources.getNeededResources()).thenReturn(Records.newRecord(Resource.class));
    Mockito.when(appResources.getReservedResources()).thenReturn(Records.newRecord(Resource.class));
    Mockito.when(appResources.getUsedResources()).thenReturn(Records.newRecord(Resource.class));
    Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(appResources);
    Mockito.when(appReport.getYarnApplicationState()).thenReturn(yarnApplicationState);
    Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(finalApplicationStatus);
    return appReport;
}

Example 18 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class ApplicationCLI method printApplicationReport.

/**
   * Prints the application report for an application id.
   * 
   * @param applicationId
   * @return exitCode
   * @throws YarnException
   */
private int printApplicationReport(String applicationId) throws YarnException, IOException {
    ApplicationReport appReport = null;
    try {
        appReport = client.getApplicationReport(ApplicationId.fromString(applicationId));
    } catch (ApplicationNotFoundException e) {
        sysout.println("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server.");
        return -1;
    }
    // Use PrintWriter.println, which uses correct platform line ending.
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter appReportStr = new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
    if (appReport != null) {
        appReportStr.println("Application Report : ");
        appReportStr.print("\tApplication-Id : ");
        appReportStr.println(appReport.getApplicationId());
        appReportStr.print("\tApplication-Name : ");
        appReportStr.println(appReport.getName());
        appReportStr.print("\tApplication-Type : ");
        appReportStr.println(appReport.getApplicationType());
        appReportStr.print("\tUser : ");
        appReportStr.println(appReport.getUser());
        appReportStr.print("\tQueue : ");
        appReportStr.println(appReport.getQueue());
        appReportStr.print("\tApplication Priority : ");
        appReportStr.println(appReport.getPriority());
        appReportStr.print("\tStart-Time : ");
        appReportStr.println(appReport.getStartTime());
        appReportStr.print("\tFinish-Time : ");
        appReportStr.println(appReport.getFinishTime());
        appReportStr.print("\tProgress : ");
        DecimalFormat formatter = new DecimalFormat("###.##%");
        String progress = formatter.format(appReport.getProgress());
        appReportStr.println(progress);
        appReportStr.print("\tState : ");
        appReportStr.println(appReport.getYarnApplicationState());
        appReportStr.print("\tFinal-State : ");
        appReportStr.println(appReport.getFinalApplicationStatus());
        appReportStr.print("\tTracking-URL : ");
        appReportStr.println(appReport.getOriginalTrackingUrl());
        appReportStr.print("\tRPC Port : ");
        appReportStr.println(appReport.getRpcPort());
        appReportStr.print("\tAM Host : ");
        appReportStr.println(appReport.getHost());
        appReportStr.print("\tAggregate Resource Allocation : ");
        ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport();
        if (usageReport != null) {
            //completed app report in the timeline server doesn't have usage report
            appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
            appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
            appReportStr.print("\tAggregate Resource Preempted : ");
            appReportStr.print(usageReport.getPreemptedMemorySeconds() + " MB-seconds, ");
            appReportStr.println(usageReport.getPreemptedVcoreSeconds() + " vcore-seconds");
        } else {
            appReportStr.println("N/A");
            appReportStr.print("\tAggregate Resource Preempted : ");
            appReportStr.println("N/A");
        }
        appReportStr.print("\tLog Aggregation Status : ");
        appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A" : appReport.getLogAggregationStatus());
        appReportStr.print("\tDiagnostics : ");
        appReportStr.println(appReport.getDiagnostics());
        appReportStr.print("\tUnmanaged Application : ");
        appReportStr.println(appReport.isUnmanagedApp());
        appReportStr.print("\tApplication Node Label Expression : ");
        appReportStr.println(appReport.getAppNodeLabelExpression());
        appReportStr.print("\tAM container Node Label Expression : ");
        appReportStr.println(appReport.getAmNodeLabelExpression());
        for (ApplicationTimeout timeout : appReport.getApplicationTimeouts().values()) {
            appReportStr.print("\tTimeoutType : " + timeout.getTimeoutType());
            appReportStr.print("\tExpiryTime : " + timeout.getExpiryTime());
            appReportStr.println("\tRemainingTime : " + timeout.getRemainingTime() + "seconds");
        }
    } else {
        appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM.");
        appReportStr.close();
        sysout.println(baos.toString("UTF-8"));
        return -1;
    }
    appReportStr.close();
    sysout.println(baos.toString("UTF-8"));
    return 0;
}

Example 19 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class TestYarnCLI method testGetApplicationReport.

@Test
public void testGetApplicationReport() throws Exception {
    for (int i = 0; i < 2; ++i) {
        ApplicationCLI cli = createAndGetAppCLI();
        ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
        ApplicationResourceUsageReport usageReport = i == 0 ? null : ApplicationResourceUsageReport.newInstance(2, 0, null, null, null, 123456, 4567, 0, 0, 1111, 2222);
        ApplicationReport newApplicationReport = ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", null, null, false, Priority.newInstance(0), "high-mem", "high-mem");
        newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
        newApplicationReport.setPriority(Priority.newInstance(0));
        ApplicationTimeout timeout = ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, "UNLIMITED", -1);
        newApplicationReport.setApplicationTimeouts(Collections.singletonMap(timeout.getTimeoutType(), timeout));
        when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(newApplicationReport);
        int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
        assertEquals(0, result);
        verify(client, times(1 + i)).getApplicationReport(applicationId);
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        PrintWriter pw = new PrintWriter(baos);
        pw.println("Application Report : ");
        pw.println("\tApplication-Id : application_1234_0005");
        pw.println("\tApplication-Name : appname");
        pw.println("\tApplication-Type : YARN");
        pw.println("\tUser : user");
        pw.println("\tQueue : queue");
        pw.println("\tApplication Priority : 0");
        pw.println("\tStart-Time : 0");
        pw.println("\tFinish-Time : 0");
        pw.println("\tProgress : 53.79%");
        pw.println("\tState : FINISHED");
        pw.println("\tFinal-State : SUCCEEDED");
        pw.println("\tTracking-URL : N/A");
        pw.println("\tRPC Port : 124");
        pw.println("\tAM Host : host");
        pw.println("\tAggregate Resource Allocation : " + (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
        pw.println("\tAggregate Resource Preempted : " + (i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds"));
        pw.println("\tLog Aggregation Status : SUCCEEDED");
        pw.println("\tDiagnostics : diagnostics");
        pw.println("\tUnmanaged Application : false");
        pw.println("\tApplication Node Label Expression : high-mem");
        pw.println("\tAM container Node Label Expression : high-mem");
        pw.print("\tTimeoutType : LIFETIME");
        pw.print("\tExpiryTime : UNLIMITED");
        pw.println("\tRemainingTime : -1seconds");
        pw.println();
        pw.close();
        String appReportStr = baos.toString("UTF-8");
        Assert.assertEquals(appReportStr, sysOutStream.toString());
        sysOutStream.reset();
        verify(sysOut, times(1 + i)).println(isA(String.class));
    }
}

Example 20 with ApplicationResourceUsageReport

use of org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport in project hadoop by apache.

the class ApplicationHistoryManagerOnTimelineStore method convertToApplicationReport.

private static ApplicationReportExt convertToApplicationReport(TimelineEntity entity, ApplicationReportField field) {
    String user = null;
    String queue = null;
    String name = null;
    String type = null;
    boolean unmanagedApplication = false;
    long createdTime = 0;
    long finishedTime = 0;
    float progress = 0.0f;
    int applicationPriority = 0;
    ApplicationAttemptId latestApplicationAttemptId = null;
    String diagnosticsInfo = null;
    FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
    YarnApplicationState state = YarnApplicationState.ACCEPTED;
    ApplicationResourceUsageReport appResources = null;
    Set<String> appTags = null;
    Map<ApplicationAccessType, String> appViewACLs = new HashMap<ApplicationAccessType, String>();
    String appNodeLabelExpression = null;
    String amNodeLabelExpression = null;
    Map<String, Object> entityInfo = entity.getOtherInfo();
    if (entityInfo != null) {
        if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) {
            user = entityInfo.get(ApplicationMetricsConstants.USER_ENTITY_INFO).toString();
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)) {
            String appViewACLsStr = entityInfo.get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO).toString();
            if (appViewACLsStr.length() > 0) {
                appViewACLs.put(ApplicationAccessType.VIEW_APP, appViewACLsStr);
            }
        }
        if (field == ApplicationReportField.USER_AND_ACLS) {
            return new ApplicationReportExt(ApplicationReport.newInstance(ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, null, progress, type, null, appTags, unmanagedApplication, Priority.newInstance(applicationPriority), appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
            queue = entityInfo.get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO).toString();
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.NAME_ENTITY_INFO)) {
            name = entityInfo.get(ApplicationMetricsConstants.NAME_ENTITY_INFO).toString();
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.TYPE_ENTITY_INFO)) {
            type = entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO).toString();
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.TYPE_ENTITY_INFO)) {
            type = entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO).toString();
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)) {
            unmanagedApplication = Boolean.parseBoolean(entityInfo.get(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO).toString());
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)) {
            applicationPriority = Integer.parseInt(entityInfo.get(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString());
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION)) {
            appNodeLabelExpression = entityInfo.get(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION).toString();
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)) {
            amNodeLabelExpression = entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION).toString();
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
            long vcoreSeconds = Long.parseLong(entityInfo.get(ApplicationMetricsConstants.APP_CPU_METRICS).toString());
            long memorySeconds = Long.parseLong(entityInfo.get(ApplicationMetricsConstants.APP_MEM_METRICS).toString());
            long preemptedMemorySeconds = Long.parseLong(entityInfo.get(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS).toString());
            long preemptedVcoreSeconds = Long.parseLong(entityInfo.get(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS).toString());
            appResources = ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, 0, 0, preemptedMemorySeconds, preemptedVcoreSeconds);
        }
        if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
            appTags = new HashSet<String>();
            Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
            if (obj != null && obj instanceof Collection<?>) {
                for (Object o : (Collection<?>) obj) {
                    if (o != null) {
                        appTags.add(o.toString());
                    }
                }
            }
        }
    }
    List<TimelineEvent> events = entity.getEvents();
    long updatedTimeStamp = 0L;
    if (events != null) {
        for (TimelineEvent event : events) {
            if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
                createdTime = event.getTimestamp();
            } else if (event.getEventType().equals(ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) {
                // before over writing.
                if (event.getTimestamp() > updatedTimeStamp) {
                    updatedTimeStamp = event.getTimestamp();
                } else {
                    continue;
                }
                Map<String, Object> eventInfo = event.getEventInfo();
                if (eventInfo == null) {
                    continue;
                }
                applicationPriority = Integer.parseInt(eventInfo.get(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString());
                queue = eventInfo.get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO).toString();
            } else if (event.getEventType().equals(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE)) {
                Map<String, Object> eventInfo = event.getEventInfo();
                if (eventInfo == null) {
                    continue;
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.STATE_EVENT_INFO)) {
                    if (!isFinalState(state)) {
                        state = YarnApplicationState.valueOf(eventInfo.get(ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
                    }
                }
            } else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
                progress = 1.0F;
                finishedTime = event.getTimestamp();
                Map<String, Object> eventInfo = event.getEventInfo();
                if (eventInfo == null) {
                    continue;
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) {
                    latestApplicationAttemptId = ApplicationAttemptId.fromString(eventInfo.get(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO).toString());
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
                    diagnosticsInfo = eventInfo.get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO).toString();
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO)) {
                    finalStatus = FinalApplicationStatus.valueOf(eventInfo.get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO).toString());
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.STATE_EVENT_INFO)) {
                    state = YarnApplicationState.valueOf(eventInfo.get(ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
                }
            }
        }
    }
    return new ApplicationReportExt(ApplicationReport.newInstance(ApplicationId.fromString(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, null, progress, type, null, appTags, unmanagedApplication, Priority.newInstance(applicationPriority), appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
}