Examples with FinalApplicationStatus org.apache.hadoop.yarn.api.records.FinalApplicationStatus used on opensource projects

Example 1 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project flink by apache.

the class YarnFlinkResourceManager method shutdownApplication.

@Override
protected void shutdownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
    // first, de-register from YARN
    FinalApplicationStatus yarnStatus = getYarnStatus(finalStatus);
    LOG.info("Unregistering application from the YARN Resource Manager");
    try {
        resourceManagerClient.unregisterApplicationMaster(yarnStatus, optionalDiagnostics, "");
    } catch (Throwable t) {
        LOG.error("Could not unregister the application master.", t);
    }
    // now shut down all our components
    try {
        resourceManagerClient.stop();
    } catch (Throwable t) {
        LOG.error("Could not cleanly shut down the Asynchronous Resource Manager Client", t);
    }
    try {
        nodeManagerClient.stop();
    } catch (Throwable t) {
        LOG.error("Could not cleanly shut down the Node Manager Client", t);
    }
}

Example 2 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project flink by apache.

the class YarnResourceManager method shutDownApplication.

@Override
protected void shutDownApplication(ApplicationStatus finalStatus, String optionalDiagnostics) {
    // first, de-register from YARN
    FinalApplicationStatus yarnStatus = getYarnStatus(finalStatus);
    LOG.info("Unregistering application from the YARN Resource Manager");
    try {
        resourceManagerClient.unregisterApplicationMaster(yarnStatus, optionalDiagnostics, "");
    } catch (Throwable t) {
        LOG.error("Could not unregister the application master.", t);
    }
}

Example 3 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project hadoop by apache.

the class RMCommunicator method doUnregistration.

@VisibleForTesting
protected void doUnregistration() throws YarnException, IOException, InterruptedException {
    FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
    JobImpl jobImpl = (JobImpl) job;
    if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
        finishState = FinalApplicationStatus.SUCCEEDED;
    } else if (jobImpl.getInternalState() == JobStateInternal.KILLED || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
        finishState = FinalApplicationStatus.KILLED;
    } else if (jobImpl.getInternalState() == JobStateInternal.FAILED || jobImpl.getInternalState() == JobStateInternal.ERROR) {
        finishState = FinalApplicationStatus.FAILED;
    }
    StringBuffer sb = new StringBuffer();
    for (String s : job.getDiagnostics()) {
        sb.append(s).append("\n");
    }
    LOG.info("Setting job diagnostics to " + sb.toString());
    String historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(), context.getApplicationID());
    LOG.info("History url is " + historyUrl);
    FinishApplicationMasterRequest request = FinishApplicationMasterRequest.newInstance(finishState, sb.toString(), historyUrl);
    try {
        while (true) {
            FinishApplicationMasterResponse response = scheduler.finishApplicationMaster(request);
            if (response.getIsUnregistered()) {
                // When excepting ClientService, other services are already stopped,
                // it is safe to let clients know the final states. ClientService
                // should wait for some time so clients have enough time to know the
                // final states.
                RunningAppContext raContext = (RunningAppContext) context;
                raContext.markSuccessfulUnregistration();
                break;
            }
            LOG.info("Waiting for application to be successfully unregistered.");
            Thread.sleep(rmPollInterval);
        }
    } catch (ApplicationMasterNotRegisteredException e) {
        // RM might have restarted or failed over and so lost the fact that AM had
        // registered before.
        register();
        doUnregistration();
    }
}

Example 4 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project hadoop by apache.

the class ApplicationMaster method finish.

@VisibleForTesting
protected boolean finish() {
    // wait for completion.
    while (!done && (numCompletedContainers.get() != numTotalContainers)) {
        try {
            Thread.sleep(200);
        } catch (InterruptedException ex) {
        }
    }
    if (timelineServiceV2Enabled) {
        publishApplicationAttemptEventOnTimelineServiceV2(DSEvent.DS_APP_ATTEMPT_END);
    } else if (timelineServiceV1Enabled) {
        publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
    }
    // and we need to release containers
    for (Thread launchThread : launchThreads) {
        try {
            launchThread.join(10000);
        } catch (InterruptedException e) {
            LOG.info("Exception thrown in thread join: " + e.getMessage());
            e.printStackTrace();
        }
    }
    // When the application completes, it should stop all running containers
    LOG.info("Application completed. Stopping running containers");
    nmClientAsync.stop();
    // When the application completes, it should send a finish application
    // signal to the RM
    LOG.info("Application completed. Signalling finish to RM");
    FinalApplicationStatus appStatus;
    String appMessage = null;
    boolean success = true;
    if (numCompletedContainers.get() - numFailedContainers.get() >= numTotalContainers) {
        appStatus = FinalApplicationStatus.SUCCEEDED;
    } else {
        appStatus = FinalApplicationStatus.FAILED;
        appMessage = "Diagnostics." + ", total=" + numTotalContainers + ", completed=" + numCompletedContainers.get() + ", allocated=" + numAllocatedContainers.get() + ", failed=" + numFailedContainers.get();
        LOG.info(appMessage);
        success = false;
    }
    try {
        amRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
    } catch (YarnException ex) {
        LOG.error("Failed to unregister application", ex);
    } catch (IOException e) {
        LOG.error("Failed to unregister application", e);
    }
    amRMClient.stop();
    // Stop Timeline Client
    if (timelineServiceV1Enabled) {
        timelineClient.stop();
    } else if (timelineServiceV2Enabled) {
        timelineV2Client.stop();
    }
    return success;
}

Example 5 with FinalApplicationStatus

use of org.apache.hadoop.yarn.api.records.FinalApplicationStatus in project hadoop by apache.

the class Client method monitorApplication.

/**
   * Monitor the submitted application for completion. 
   * Kill application if time expires. 
   * @param appId Application Id of application to be monitored
   * @return true if application completed successfully
   * @throws YarnException
   * @throws IOException
   */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
    while (true) {
        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }
        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);
        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }
}