Examples with ApplicationTimeout org.apache.hadoop.yarn.api.records.ApplicationTimeout used on opensource projects

Example 1 with ApplicationTimeout

use of org.apache.hadoop.yarn.api.records.ApplicationTimeout in project hadoop by apache.

the class TestApplicationLifetimeMonitor method testApplicationLifetimeMonitor.

@Test(timeout = 60000)
public void testApplicationLifetimeMonitor() throws Exception {
    MockRM rm = null;
    try {
        rm = new MockRM(conf);
        rm.start();
        Priority appPriority = Priority.newInstance(0);
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * 1024);
        Map<ApplicationTimeoutType, Long> timeouts = new HashMap<ApplicationTimeoutType, Long>();
        timeouts.put(ApplicationTimeoutType.LIFETIME, 10L);
        RMApp app1 = rm.submitApp(1024, appPriority, timeouts);
        // 20L seconds
        timeouts.put(ApplicationTimeoutType.LIFETIME, 20L);
        RMApp app2 = rm.submitApp(1024, appPriority, timeouts);
        nm1.nodeHeartbeat(true);
        // Send launch Event
        MockAM am1 = rm.sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
        am1.registerAppAttempt();
        rm.waitForState(app1.getApplicationId(), RMAppState.KILLED);
        Assert.assertTrue("Application killed before lifetime value", (System.currentTimeMillis() - app1.getSubmitTime()) > 10000);
        Map<ApplicationTimeoutType, String> updateTimeout = new HashMap<ApplicationTimeoutType, String>();
        long newLifetime = 10L;
        // update 10L seconds more to timeout
        String formatISO8601 = Times.formatISO8601(System.currentTimeMillis() + newLifetime * 1000);
        updateTimeout.put(ApplicationTimeoutType.LIFETIME, formatISO8601);
        UpdateApplicationTimeoutsRequest request = UpdateApplicationTimeoutsRequest.newInstance(app2.getApplicationId(), updateTimeout);
        Map<ApplicationTimeoutType, Long> applicationTimeouts = app2.getApplicationTimeouts();
        // has old timeout time
        long beforeUpdate = applicationTimeouts.get(ApplicationTimeoutType.LIFETIME);
        // update app2 lifetime to new time i.e now + timeout
        rm.getRMContext().getClientRMService().updateApplicationTimeouts(request);
        applicationTimeouts = app2.getApplicationTimeouts();
        long afterUpdate = applicationTimeouts.get(ApplicationTimeoutType.LIFETIME);
        Assert.assertTrue("Application lifetime value not updated", afterUpdate > beforeUpdate);
        // verify for application report.
        RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
        GetApplicationReportRequest appRequest = recordFactory.newRecordInstance(GetApplicationReportRequest.class);
        appRequest.setApplicationId(app2.getApplicationId());
        Map<ApplicationTimeoutType, ApplicationTimeout> appTimeouts = rm.getRMContext().getClientRMService().getApplicationReport(appRequest).getApplicationReport().getApplicationTimeouts();
        Assert.assertTrue("Application Timeout are empty.", !appTimeouts.isEmpty());
        ApplicationTimeout timeout = appTimeouts.get(ApplicationTimeoutType.LIFETIME);
        Assert.assertEquals("Application timeout string is incorrect.", formatISO8601, timeout.getExpiryTime());
        Assert.assertTrue("Application remaining time is incorrect", timeout.getRemainingTime() > 0);
        rm.waitForState(app2.getApplicationId(), RMAppState.KILLED);
        // verify for app killed with updated lifetime
        Assert.assertTrue("Application killed before lifetime value", app2.getFinishTime() > afterUpdate);
    } finally {
        stopRM(rm);
    }
}

Example 2 with ApplicationTimeout

use of org.apache.hadoop.yarn.api.records.ApplicationTimeout in project hadoop by apache.

the class ApplicationReportPBImpl method initApplicationTimeout.

private void initApplicationTimeout() {
    if (this.applicationTimeouts != null) {
        return;
    }
    ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
    List<AppTimeoutsMapProto> lists = p.getAppTimeoutsList();
    this.applicationTimeouts = new HashMap<ApplicationTimeoutType, ApplicationTimeout>(lists.size());
    for (AppTimeoutsMapProto timeoutProto : lists) {
        this.applicationTimeouts.put(ProtoUtils.convertFromProtoFormat(timeoutProto.getApplicationTimeoutType()), convertFromProtoFormat(timeoutProto.getApplicationTimeout()));
    }
}

Example 3 with ApplicationTimeout

use of org.apache.hadoop.yarn.api.records.ApplicationTimeout in project hadoop by apache.

the class TestYarnCLI method testUpdateApplicationTimeout.

@Test(timeout = 60000)
public void testUpdateApplicationTimeout() throws Exception {
    ApplicationCLI cli = createAndGetAppCLI();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 6);
    ApplicationReport appReport = ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.53789f, "YARN", null);
    ApplicationTimeout timeout = ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1);
    appReport.setApplicationTimeouts(Collections.singletonMap(timeout.getTimeoutType(), timeout));
    when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(appReport);
    int result = cli.run(new String[] { "application", "-appId", applicationId.toString(), "-updateLifetime", "10" });
    Assert.assertEquals(result, 0);
    verify(client).updateApplicationTimeouts(any(UpdateApplicationTimeoutsRequest.class));
}

Example 4 with ApplicationTimeout

use of org.apache.hadoop.yarn.api.records.ApplicationTimeout in project hadoop by apache.

the class RMAppImpl method createAndGetApplicationReport.

@Override
public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
    this.readLock.lock();
    try {
        ApplicationAttemptId currentApplicationAttemptId = null;
        org.apache.hadoop.yarn.api.records.Token clientToAMToken = null;
        String trackingUrl = UNAVAILABLE;
        String host = UNAVAILABLE;
        String origTrackingUrl = UNAVAILABLE;
        LogAggregationStatus logAggregationStatus = null;
        int rpcPort = -1;
        ApplicationResourceUsageReport appUsageReport = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
        FinalApplicationStatus finishState = getFinalApplicationStatus();
        String diags = UNAVAILABLE;
        float progress = 0.0f;
        org.apache.hadoop.yarn.api.records.Token amrmToken = null;
        if (allowAccess) {
            trackingUrl = getDefaultProxyTrackingUrl();
            if (this.currentAttempt != null) {
                currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
                trackingUrl = this.currentAttempt.getTrackingUrl();
                origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
                if (UserGroupInformation.isSecurityEnabled()) {
                    // get a token so the client can communicate with the app attempt
                    // NOTE: token may be unavailable if the attempt is not running
                    Token<ClientToAMTokenIdentifier> attemptClientToAMToken = this.currentAttempt.createClientToken(clientUserName);
                    if (attemptClientToAMToken != null) {
                        clientToAMToken = BuilderUtils.newClientToAMToken(attemptClientToAMToken.getIdentifier(), attemptClientToAMToken.getKind().toString(), attemptClientToAMToken.getPassword(), attemptClientToAMToken.getService().toString());
                    }
                }
                host = this.currentAttempt.getHost();
                rpcPort = this.currentAttempt.getRpcPort();
                appUsageReport = currentAttempt.getApplicationResourceUsageReport();
                progress = currentAttempt.getProgress();
                logAggregationStatus = this.getLogAggregationStatusForAppReport();
            }
            //if the diagnostics is not already set get it from attempt
            diags = getDiagnostics().toString();
            if (currentAttempt != null && currentAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
                if (getApplicationSubmissionContext().getUnmanagedAM() && clientUserName != null && getUser().equals(clientUserName)) {
                    Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
                    if (token != null) {
                        amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString());
                    }
                }
            }
            RMAppMetrics rmAppMetrics = getRMAppMetrics();
            appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
            appUsageReport.setPreemptedMemorySeconds(rmAppMetrics.getPreemptedMemorySeconds());
            appUsageReport.setPreemptedVcoreSeconds(rmAppMetrics.getPreemptedVcoreSeconds());
        }
        if (currentApplicationAttemptId == null) {
            currentApplicationAttemptId = BuilderUtils.newApplicationAttemptId(this.applicationId, DUMMY_APPLICATION_ATTEMPT_NUMBER);
        }
        ApplicationReport report = BuilderUtils.newApplicationReport(this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, amrmToken, applicationTags, this.getApplicationPriority());
        report.setLogAggregationStatus(logAggregationStatus);
        report.setUnmanagedApp(submissionContext.getUnmanagedAM());
        report.setAppNodeLabelExpression(getAppNodeLabelExpression());
        report.setAmNodeLabelExpression(getAmNodeLabelExpression());
        ApplicationTimeout timeout = ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, UNLIMITED, UNKNOWN);
        // are supported in YARN-5692, the below logic need to be changed.
        if (!this.applicationTimeouts.isEmpty()) {
            long timeoutInMillis = applicationTimeouts.get(ApplicationTimeoutType.LIFETIME).longValue();
            timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
            if (isAppInCompletedStates()) {
                // if application configured with timeout and finished before timeout
                // happens then remaining time should not be calculated.
                timeout.setRemainingTime(0);
            } else {
                timeout.setRemainingTime(Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0));
            }
        }
        report.setApplicationTimeouts(Collections.singletonMap(timeout.getTimeoutType(), timeout));
        return report;
    } finally {
        this.readLock.unlock();
    }
}

Example 5 with ApplicationTimeout

use of org.apache.hadoop.yarn.api.records.ApplicationTimeout in project hadoop by apache.

the class ApplicationCLI method printApplicationReport.

/**
   * Prints the application report for an application id.
   * 
   * @param applicationId
   * @return exitCode
   * @throws YarnException
   */
private int printApplicationReport(String applicationId) throws YarnException, IOException {
    ApplicationReport appReport = null;
    try {
        appReport = client.getApplicationReport(ApplicationId.fromString(applicationId));
    } catch (ApplicationNotFoundException e) {
        sysout.println("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server.");
        return -1;
    }
    // Use PrintWriter.println, which uses correct platform line ending.
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter appReportStr = new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
    if (appReport != null) {
        appReportStr.println("Application Report : ");
        appReportStr.print("\tApplication-Id : ");
        appReportStr.println(appReport.getApplicationId());
        appReportStr.print("\tApplication-Name : ");
        appReportStr.println(appReport.getName());
        appReportStr.print("\tApplication-Type : ");
        appReportStr.println(appReport.getApplicationType());
        appReportStr.print("\tUser : ");
        appReportStr.println(appReport.getUser());
        appReportStr.print("\tQueue : ");
        appReportStr.println(appReport.getQueue());
        appReportStr.print("\tApplication Priority : ");
        appReportStr.println(appReport.getPriority());
        appReportStr.print("\tStart-Time : ");
        appReportStr.println(appReport.getStartTime());
        appReportStr.print("\tFinish-Time : ");
        appReportStr.println(appReport.getFinishTime());
        appReportStr.print("\tProgress : ");
        DecimalFormat formatter = new DecimalFormat("###.##%");
        String progress = formatter.format(appReport.getProgress());
        appReportStr.println(progress);
        appReportStr.print("\tState : ");
        appReportStr.println(appReport.getYarnApplicationState());
        appReportStr.print("\tFinal-State : ");
        appReportStr.println(appReport.getFinalApplicationStatus());
        appReportStr.print("\tTracking-URL : ");
        appReportStr.println(appReport.getOriginalTrackingUrl());
        appReportStr.print("\tRPC Port : ");
        appReportStr.println(appReport.getRpcPort());
        appReportStr.print("\tAM Host : ");
        appReportStr.println(appReport.getHost());
        appReportStr.print("\tAggregate Resource Allocation : ");
        ApplicationResourceUsageReport usageReport = appReport.getApplicationResourceUsageReport();
        if (usageReport != null) {
            //completed app report in the timeline server doesn't have usage report
            appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
            appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
            appReportStr.print("\tAggregate Resource Preempted : ");
            appReportStr.print(usageReport.getPreemptedMemorySeconds() + " MB-seconds, ");
            appReportStr.println(usageReport.getPreemptedVcoreSeconds() + " vcore-seconds");
        } else {
            appReportStr.println("N/A");
            appReportStr.print("\tAggregate Resource Preempted : ");
            appReportStr.println("N/A");
        }
        appReportStr.print("\tLog Aggregation Status : ");
        appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A" : appReport.getLogAggregationStatus());
        appReportStr.print("\tDiagnostics : ");
        appReportStr.println(appReport.getDiagnostics());
        appReportStr.print("\tUnmanaged Application : ");
        appReportStr.println(appReport.isUnmanagedApp());
        appReportStr.print("\tApplication Node Label Expression : ");
        appReportStr.println(appReport.getAppNodeLabelExpression());
        appReportStr.print("\tAM container Node Label Expression : ");
        appReportStr.println(appReport.getAmNodeLabelExpression());
        for (ApplicationTimeout timeout : appReport.getApplicationTimeouts().values()) {
            appReportStr.print("\tTimeoutType : " + timeout.getTimeoutType());
            appReportStr.print("\tExpiryTime : " + timeout.getExpiryTime());
            appReportStr.println("\tRemainingTime : " + timeout.getRemainingTime() + "seconds");
        }
    } else {
        appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM.");
        appReportStr.close();
        sysout.println(baos.toString("UTF-8"));
        return -1;
    }
    appReportStr.close();
    sysout.println(baos.toString("UTF-8"));
    return 0;
}