Examples with FinalApplicationStatus org.apache.hadoop.yarn.api.records.FinalApplicationStatus used on opensource projects

Example 6 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project hadoop by apache.

the class TestRMAppAttemptTransitions method testFinalSavingToFinishedWithExpire.

// While attempt is at FINAL_SAVING, Expire event may come before
// Attempt_Saved event, we stay on FINAL_SAVING on Expire event and then
// directly jump from FINAL_SAVING to FINISHED state on Attempt_Saved event.
@Test
public void testFinalSavingToFinishedWithExpire() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
    String trackingUrl = "mytrackingurl";
    String diagnostics = "Successssseeeful";
    applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus, diagnostics));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    // Expire event comes before Attempt_saved event.
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    // send attempt_saved
    sendAttemptUpdateSavedEvent(applicationAttempt);
    testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl, diagnostics, 0, false);
}

Example 7 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project hadoop by apache.

the class ApplicationMaster method finish.

@VisibleForTesting
protected boolean finish() {
    // wait for completion.
    while (!done && (numCompletedContainers.get() != numTotalContainers)) {
        try {
            Thread.sleep(200);
        } catch (InterruptedException ex) {
        }
    }
    if (timelineServiceV2Enabled) {
        publishApplicationAttemptEventOnTimelineServiceV2(DSEvent.DS_APP_ATTEMPT_END);
    } else if (timelineServiceV1Enabled) {
        publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
    }
    // and we need to release containers
    for (Thread launchThread : launchThreads) {
        try {
            launchThread.join(10000);
        } catch (InterruptedException e) {
            LOG.info("Exception thrown in thread join: " + e.getMessage());
            e.printStackTrace();
        }
    }
    // When the application completes, it should stop all running containers
    LOG.info("Application completed. Stopping running containers");
    nmClientAsync.stop();
    // When the application completes, it should send a finish application
    // signal to the RM
    LOG.info("Application completed. Signalling finish to RM");
    FinalApplicationStatus appStatus;
    String appMessage = null;
    boolean success = true;
    if (numCompletedContainers.get() - numFailedContainers.get() >= numTotalContainers) {
        appStatus = FinalApplicationStatus.SUCCEEDED;
    } else {
        appStatus = FinalApplicationStatus.FAILED;
        appMessage = "Diagnostics." + ", total=" + numTotalContainers + ", completed=" + numCompletedContainers.get() + ", allocated=" + numAllocatedContainers.get() + ", failed=" + numFailedContainers.get();
        LOG.info(appMessage);
        success = false;
    }
    try {
        amRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
    } catch (YarnException ex) {
        LOG.error("Failed to unregister application", ex);
    } catch (IOException e) {
        LOG.error("Failed to unregister application", e);
    }
    amRMClient.stop();
    // Stop Timeline Client
    if (timelineServiceV1Enabled) {
        timelineClient.stop();
    } else if (timelineServiceV2Enabled) {
        timelineV2Client.stop();
    }
    return success;
}

Example 8 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project hadoop by apache.

the class Client method monitorApplication.

/**
   * Monitor the submitted application for completion. 
   * Kill application if time expires. 
   * @param appId Application Id of application to be monitored
   * @return true if application completed successfully
   * @throws YarnException
   * @throws IOException
   */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
    while (true) {
        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }
        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);
        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }
}

Example 9 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project hadoop by apache.

the class RMAppAttemptImpl method rememberTargetTransitionsAndStoreState.

private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, Object transitionToDo, RMAppAttemptState targetFinalState, RMAppAttemptState stateToBeStored) {
    rememberTargetTransitions(event, transitionToDo, targetFinalState);
    stateBeforeFinalSaving = getState();
    // As of today, finalState, diagnostics, final-tracking-url and
    // finalAppStatus are the only things that we store into the StateStore
    // AFTER the initial saving on app-attempt-start
    // These fields can be visible from outside only after they are saved in
    // StateStore
    String diags = null;
    // don't leave the tracking URL pointing to a non-existent AM
    if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
        setTrackingUrlToAHSPage(stateToBeStored);
    } else {
        setTrackingUrlToRMAppPage(stateToBeStored);
    }
    String finalTrackingUrl = getOriginalTrackingUrl();
    FinalApplicationStatus finalStatus = null;
    int exitStatus = ContainerExitStatus.INVALID;
    switch(event.getType()) {
        case LAUNCH_FAILED:
            diags = event.getDiagnosticMsg();
            break;
        case REGISTERED:
            diags = getUnexpectedAMRegisteredDiagnostics();
            break;
        case UNREGISTERED:
            RMAppAttemptUnregistrationEvent unregisterEvent = (RMAppAttemptUnregistrationEvent) event;
            diags = unregisterEvent.getDiagnosticMsg();
            // reset finalTrackingUrl to url sent by am
            finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
            finalStatus = unregisterEvent.getFinalApplicationStatus();
            break;
        case CONTAINER_FINISHED:
            RMAppAttemptContainerFinishedEvent finishEvent = (RMAppAttemptContainerFinishedEvent) event;
            diags = getAMContainerCrashedDiagnostics(finishEvent);
            exitStatus = finishEvent.getContainerStatus().getExitStatus();
            break;
        case KILL:
            break;
        case FAIL:
            diags = event.getDiagnosticMsg();
            break;
        case EXPIRE:
            diags = getAMExpiredDiagnostics(event);
            break;
        default:
            break;
    }
    AggregateAppResourceUsage resUsage = this.attemptMetrics.getAggregateAppResourceUsage();
    RMStateStore rmStore = rmContext.getStateStore();
    setFinishTime(System.currentTimeMillis());
    ApplicationAttemptStateData attemptState = ApplicationAttemptStateData.newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, stateToBeStored, finalTrackingUrl, diags, finalStatus, exitStatus, getFinishTime(), resUsage.getMemorySeconds(), resUsage.getVcoreSeconds(), this.attemptMetrics.getPreemptedMemory(), this.attemptMetrics.getPreemptedVcore());
    LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus);
    rmStore.updateApplicationAttemptState(attemptState);
}

Example 10 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project hadoop by apache.

the class RMAppImpl method createAndGetApplicationReport.

@Override
public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
    this.readLock.lock();
    try {
        ApplicationAttemptId currentApplicationAttemptId = null;
        org.apache.hadoop.yarn.api.records.Token clientToAMToken = null;
        String trackingUrl = UNAVAILABLE;
        String host = UNAVAILABLE;
        String origTrackingUrl = UNAVAILABLE;
        LogAggregationStatus logAggregationStatus = null;
        int rpcPort = -1;
        ApplicationResourceUsageReport appUsageReport = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
        FinalApplicationStatus finishState = getFinalApplicationStatus();
        String diags = UNAVAILABLE;
        float progress = 0.0f;
        org.apache.hadoop.yarn.api.records.Token amrmToken = null;
        if (allowAccess) {
            trackingUrl = getDefaultProxyTrackingUrl();
            if (this.currentAttempt != null) {
                currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
                trackingUrl = this.currentAttempt.getTrackingUrl();
                origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
                if (UserGroupInformation.isSecurityEnabled()) {
                    // get a token so the client can communicate with the app attempt
                    // NOTE: token may be unavailable if the attempt is not running
                    Token<ClientToAMTokenIdentifier> attemptClientToAMToken = this.currentAttempt.createClientToken(clientUserName);
                    if (attemptClientToAMToken != null) {
                        clientToAMToken = BuilderUtils.newClientToAMToken(attemptClientToAMToken.getIdentifier(), attemptClientToAMToken.getKind().toString(), attemptClientToAMToken.getPassword(), attemptClientToAMToken.getService().toString());
                    }
                }
                host = this.currentAttempt.getHost();
                rpcPort = this.currentAttempt.getRpcPort();
                appUsageReport = currentAttempt.getApplicationResourceUsageReport();
                progress = currentAttempt.getProgress();
                logAggregationStatus = this.getLogAggregationStatusForAppReport();
            }
            //if the diagnostics is not already set get it from attempt
            diags = getDiagnostics().toString();
            if (currentAttempt != null && currentAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
                if (getApplicationSubmissionContext().getUnmanagedAM() && clientUserName != null && getUser().equals(clientUserName)) {
                    Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
                    if (token != null) {
                        amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(), token.getKind().toString(), token.getPassword(), token.getService().toString());
                    }
                }
            }
            RMAppMetrics rmAppMetrics = getRMAppMetrics();
            appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
            appUsageReport.setPreemptedMemorySeconds(rmAppMetrics.getPreemptedMemorySeconds());
            appUsageReport.setPreemptedVcoreSeconds(rmAppMetrics.getPreemptedVcoreSeconds());
        }
        if (currentApplicationAttemptId == null) {
            currentApplicationAttemptId = BuilderUtils.newApplicationAttemptId(this.applicationId, DUMMY_APPLICATION_ATTEMPT_NUMBER);
        }
        ApplicationReport report = BuilderUtils.newApplicationReport(this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, amrmToken, applicationTags, this.getApplicationPriority());
        report.setLogAggregationStatus(logAggregationStatus);
        report.setUnmanagedApp(submissionContext.getUnmanagedAM());
        report.setAppNodeLabelExpression(getAppNodeLabelExpression());
        report.setAmNodeLabelExpression(getAmNodeLabelExpression());
        ApplicationTimeout timeout = ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, UNLIMITED, UNKNOWN);
        // are supported in YARN-5692, the below logic need to be changed.
        if (!this.applicationTimeouts.isEmpty()) {
            long timeoutInMillis = applicationTimeouts.get(ApplicationTimeoutType.LIFETIME).longValue();
            timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
            if (isAppInCompletedStates()) {
                // if application configured with timeout and finished before timeout
                // happens then remaining time should not be calculated.
                timeout.setRemainingTime(0);
            } else {
                timeout.setRemainingTime(Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0));
            }
        }
        report.setApplicationTimeouts(Collections.singletonMap(timeout.getTimeoutType(), timeout));
        return report;
    } finally {
        this.readLock.unlock();
    }
}