Search in sources :

Example 26 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class LogsCLI method runCommand.

private int runCommand(String[] args) throws Exception {
    Options opts = createCommandOpts();
    Options printOpts = createPrintOpts(opts);
    if (args.length < 1) {
        printHelpMessage(printOpts);
        return -1;
    }
    if (args[0].equals("-help")) {
        printHelpMessage(printOpts);
        return 0;
    }
    CommandLineParser parser = new GnuParser();
    String appIdStr = null;
    String containerIdStr = null;
    String nodeAddress = null;
    String appOwner = null;
    boolean getAMContainerLogs = false;
    boolean nodesList = false;
    boolean showApplicationLogInfo = false;
    boolean showContainerLogInfo = false;
    boolean useRegex = false;
    String[] logFiles = null;
    String[] logFilesRegex = null;
    List<String> amContainersList = new ArrayList<String>();
    String localDir = null;
    long bytes = Long.MAX_VALUE;
    try {
        CommandLine commandLine = parser.parse(opts, args, false);
        appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION);
        containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION);
        nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
        appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
        getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
        nodesList = commandLine.hasOption(LIST_NODES_OPTION);
        localDir = commandLine.getOptionValue(OUT_OPTION);
        showApplicationLogInfo = commandLine.hasOption(SHOW_APPLICATION_LOG_INFO);
        showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
        if (getAMContainerLogs) {
            try {
                amContainersList = parseAMContainer(commandLine, printOpts);
            } catch (NumberFormatException ex) {
                System.err.println(ex.getMessage());
                return -1;
            }
        }
        if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) {
            logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION);
        }
        if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_REGEX_OPTION)) {
            logFilesRegex = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_REGEX_OPTION);
            useRegex = true;
        }
        if (commandLine.hasOption(SIZE_OPTION)) {
            bytes = Long.parseLong(commandLine.getOptionValue(SIZE_OPTION));
        }
    } catch (ParseException e) {
        System.err.println("options parsing failed: " + e.getMessage());
        printHelpMessage(printOpts);
        return -1;
    }
    if (appIdStr == null && containerIdStr == null) {
        System.err.println("Both applicationId and containerId are missing, " + " one of them must be specified.");
        printHelpMessage(printOpts);
        return -1;
    }
    ApplicationId appId = null;
    if (appIdStr != null) {
        try {
            appId = ApplicationId.fromString(appIdStr);
        } catch (Exception e) {
            System.err.println("Invalid ApplicationId specified");
            return -1;
        }
    }
    if (containerIdStr != null) {
        try {
            ContainerId containerId = ContainerId.fromString(containerIdStr);
            if (appId == null) {
                appId = containerId.getApplicationAttemptId().getApplicationId();
            } else if (!containerId.getApplicationAttemptId().getApplicationId().equals(appId)) {
                System.err.println("The Application:" + appId + " does not have the container:" + containerId);
                return -1;
            }
        } catch (Exception e) {
            System.err.println("Invalid ContainerId specified");
            return -1;
        }
    }
    if (showApplicationLogInfo && showContainerLogInfo) {
        System.err.println("Invalid options. Can only accept one of " + "show_application_log_info/show_container_log_info.");
        return -1;
    }
    if (logFiles != null && logFiles.length > 0 && logFilesRegex != null && logFilesRegex.length > 0) {
        System.err.println("Invalid options. Can only accept one of " + "log_files/log_files_pattern.");
        return -1;
    }
    if (localDir != null) {
        File file = new File(localDir);
        if (file.exists() && file.isFile()) {
            System.err.println("Invalid value for -out option. " + "Please provide a directory.");
            return -1;
        }
    }
    LogCLIHelpers logCliHelper = new LogCLIHelpers();
    logCliHelper.setConf(getConf());
    YarnApplicationState appState = YarnApplicationState.NEW;
    ApplicationReport appReport = null;
    try {
        appReport = getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
        if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) {
            System.err.println("Logs are not available right now.");
            return -1;
        }
    } catch (IOException | YarnException e) {
        // If we can not get appReport from either RM or ATS
        // We will assume that this app has already finished.
        appState = YarnApplicationState.FINISHED;
        System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem.");
    }
    if (appOwner == null || appOwner.isEmpty()) {
        appOwner = guessAppOwner(appReport, appId);
        if (appOwner == null) {
            System.err.println("Can not find the appOwner. " + "Please specify the correct appOwner");
            System.err.println("Could not locate application logs for " + appId);
            return -1;
        }
    }
    Set<String> logs = new HashSet<String>();
    if (fetchAllLogFiles(logFiles, logFilesRegex)) {
        logs.add("ALL");
    } else if (logFiles != null && logFiles.length > 0) {
        logs.addAll(Arrays.asList(logFiles));
    } else if (logFilesRegex != null && logFilesRegex.length > 0) {
        logs.addAll(Arrays.asList(logFilesRegex));
    }
    ContainerLogsRequest request = new ContainerLogsRequest(appId, isApplicationFinished(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes, null);
    if (showContainerLogInfo) {
        return showContainerLogInfo(request, logCliHelper);
    }
    if (nodesList) {
        return showNodeLists(request, logCliHelper);
    }
    if (showApplicationLogInfo) {
        return showApplicationLogInfo(request, logCliHelper);
    }
    // To get am logs
    if (getAMContainerLogs) {
        return fetchAMContainerLogs(request, amContainersList, logCliHelper, useRegex);
    }
    int resultCode = 0;
    if (containerIdStr != null) {
        return fetchContainerLogs(request, logCliHelper, useRegex);
    } else {
        if (nodeAddress == null) {
            resultCode = fetchApplicationLogs(request, logCliHelper, useRegex);
        } else {
            System.err.println("Should at least provide ContainerId!");
            printHelpMessage(printOpts);
            resultCode = -1;
        }
    }
    return resultCode;
}
Also used : Options(org.apache.commons.cli.Options) GnuParser(org.apache.commons.cli.GnuParser) ArrayList(java.util.ArrayList) YarnApplicationState(org.apache.hadoop.yarn.api.records.YarnApplicationState) ContainerLogsRequest(org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest) YarnException(org.apache.hadoop.yarn.exceptions.YarnException) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) CommandLineParser(org.apache.commons.cli.CommandLineParser) HashSet(java.util.HashSet) IOException(java.io.IOException) ParseException(org.apache.commons.cli.ParseException) UniformInterfaceException(com.sun.jersey.api.client.UniformInterfaceException) YarnException(org.apache.hadoop.yarn.exceptions.YarnException) IOException(java.io.IOException) JSONException(org.codehaus.jettison.json.JSONException) ClientHandlerException(com.sun.jersey.api.client.ClientHandlerException) ApplicationReport(org.apache.hadoop.yarn.api.records.ApplicationReport) CommandLine(org.apache.commons.cli.CommandLine) LogCLIHelpers(org.apache.hadoop.yarn.logaggregation.LogCLIHelpers) ParseException(org.apache.commons.cli.ParseException) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) File(java.io.File)

Example 27 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class ApplicationCLI method listApplications.

/**
   * Lists the applications matching the given application Types, application
   * States and application Tags present in the Resource Manager.
   * 
   * @param appTypes
   * @param appStates
   * @param appTags
   * @throws YarnException
   * @throws IOException
   */
private void listApplications(Set<String> appTypes, EnumSet<YarnApplicationState> appStates, Set<String> appTags) throws YarnException, IOException {
    PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
    if (allAppStates) {
        for (YarnApplicationState appState : YarnApplicationState.values()) {
            appStates.add(appState);
        }
    } else {
        if (appStates.isEmpty()) {
            appStates.add(YarnApplicationState.RUNNING);
            appStates.add(YarnApplicationState.ACCEPTED);
            appStates.add(YarnApplicationState.SUBMITTED);
        }
    }
    List<ApplicationReport> appsReport = client.getApplications(appTypes, appStates, appTags);
    writer.println("Total number of applications (application-types: " + appTypes + ", states: " + appStates + " and tags: " + appTags + ")" + ":" + appsReport.size());
    writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name", "Application-Type", "User", "Queue", "State", "Final-State", "Progress", "Tracking-URL");
    for (ApplicationReport appReport : appsReport) {
        DecimalFormat formatter = new DecimalFormat("###.##%");
        String progress = formatter.format(appReport.getProgress());
        writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), appReport.getName(), appReport.getApplicationType(), appReport.getUser(), appReport.getQueue(), appReport.getYarnApplicationState(), appReport.getFinalApplicationStatus(), progress, appReport.getOriginalTrackingUrl());
    }
    writer.flush();
}
Also used : ApplicationReport(org.apache.hadoop.yarn.api.records.ApplicationReport) DecimalFormat(java.text.DecimalFormat) YarnApplicationState(org.apache.hadoop.yarn.api.records.YarnApplicationState) OutputStreamWriter(java.io.OutputStreamWriter) PrintWriter(java.io.PrintWriter)

Example 28 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class ApplicationCLI method run.

@Override
public int run(String[] args) throws Exception {
    Options opts = new Options();
    String title = null;
    if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION)) {
        title = APPLICATION;
        opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
        opts.addOption(LIST_CMD, false, "List applications. " + "Supports optional use of -appTypes to filter applications " + "based on application type, -appStates to filter applications " + "based on application state and -appTags to filter applications " + "based on application tag.");
        opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a " + "different queue. Deprecated command. Use 'changeQueue' instead.");
        opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to" + " specify which queue to move an application to.");
        opts.addOption(HELP_CMD, false, "Displays help for all commands.");
        Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " + "filter applications based on " + "input comma-separated list of application types.");
        appTypeOpt.setValueSeparator(',');
        appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
        appTypeOpt.setArgName("Types");
        opts.addOption(appTypeOpt);
        Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " + "to filter applications based on input comma-separated list of " + "application states. " + getAllValidApplicationStates());
        appStateOpt.setValueSeparator(',');
        appStateOpt.setArgs(Option.UNLIMITED_VALUES);
        appStateOpt.setArgName("States");
        opts.addOption(appStateOpt);
        Option appTagOpt = new Option(APP_TAG_CMD, true, "Works with -list to " + "filter applications based on input comma-separated list of " + "application tags.");
        appTagOpt.setValueSeparator(',');
        appTagOpt.setArgs(Option.UNLIMITED_VALUES);
        appTagOpt.setArgName("Tags");
        opts.addOption(appTagOpt);
        opts.addOption(APP_ID, true, "Specify Application Id to be operated");
        opts.addOption(UPDATE_PRIORITY, true, "update priority of an application. ApplicationId can be" + " passed using 'appId' option.");
        opts.addOption(UPDATE_LIFETIME, true, "update timeout of an application from NOW. ApplicationId can be" + " passed using 'appId' option. Timeout value is in seconds.");
        opts.addOption(CHANGE_APPLICATION_QUEUE, true, "Moves application to a new queue. ApplicationId can be" + " passed using 'appId' option. 'movetoqueue' command is" + " deprecated, this new command 'changeQueue' performs same" + " functionality.");
        Option killOpt = new Option(KILL_CMD, true, "Kills the application. " + "Set of applications can be provided separated with space");
        killOpt.setValueSeparator(' ');
        killOpt.setArgs(Option.UNLIMITED_VALUES);
        killOpt.setArgName("Application ID");
        opts.addOption(killOpt);
        opts.getOption(MOVE_TO_QUEUE_CMD).setArgName("Application ID");
        opts.getOption(QUEUE_CMD).setArgName("Queue Name");
        opts.getOption(STATUS_CMD).setArgName("Application ID");
        opts.getOption(APP_ID).setArgName("Application ID");
        opts.getOption(UPDATE_PRIORITY).setArgName("Priority");
        opts.getOption(UPDATE_LIFETIME).setArgName("Timeout");
        opts.getOption(CHANGE_APPLICATION_QUEUE).setArgName("Queue Name");
    } else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
        title = APPLICATION_ATTEMPT;
        opts.addOption(STATUS_CMD, true, "Prints the status of the application attempt.");
        opts.addOption(LIST_CMD, true, "List application attempts for application.");
        opts.addOption(FAIL_CMD, true, "Fails application attempt.");
        opts.addOption(HELP_CMD, false, "Displays help for all commands.");
        opts.getOption(STATUS_CMD).setArgName("Application Attempt ID");
        opts.getOption(LIST_CMD).setArgName("Application ID");
        opts.getOption(FAIL_CMD).setArgName("Application Attempt ID");
    } else if (args.length > 0 && args[0].equalsIgnoreCase(CONTAINER)) {
        title = CONTAINER;
        opts.addOption(STATUS_CMD, true, "Prints the status of the container.");
        opts.addOption(LIST_CMD, true, "List containers for application attempt.");
        opts.addOption(HELP_CMD, false, "Displays help for all commands.");
        opts.getOption(STATUS_CMD).setArgName("Container ID");
        opts.getOption(LIST_CMD).setArgName("Application Attempt ID");
        opts.addOption(SIGNAL_CMD, true, "Signal the container. The available signal commands are " + java.util.Arrays.asList(SignalContainerCommand.values()) + " Default command is OUTPUT_THREAD_DUMP.");
        opts.getOption(SIGNAL_CMD).setArgName("container ID [signal command]");
        opts.getOption(SIGNAL_CMD).setArgs(3);
    }
    int exitCode = -1;
    CommandLine cliParser = null;
    try {
        cliParser = new GnuParser().parse(opts, args);
    } catch (MissingArgumentException ex) {
        sysout.println("Missing argument for options");
        printUsage(title, opts);
        return exitCode;
    }
    if (cliParser.hasOption(STATUS_CMD)) {
        if (args.length != 3) {
            printUsage(title, opts);
            return exitCode;
        }
        if (args[0].equalsIgnoreCase(APPLICATION)) {
            exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
        } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
            exitCode = printApplicationAttemptReport(cliParser.getOptionValue(STATUS_CMD));
        } else if (args[0].equalsIgnoreCase(CONTAINER)) {
            exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD));
        }
        return exitCode;
    } else if (cliParser.hasOption(LIST_CMD)) {
        if (args[0].equalsIgnoreCase(APPLICATION)) {
            allAppStates = false;
            Set<String> appTypes = new HashSet<String>();
            if (cliParser.hasOption(APP_TYPE_CMD)) {
                String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
                if (types != null) {
                    for (String type : types) {
                        if (!type.trim().isEmpty()) {
                            appTypes.add(StringUtils.toUpperCase(type).trim());
                        }
                    }
                }
            }
            EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);
            if (cliParser.hasOption(APP_STATE_CMD)) {
                String[] states = cliParser.getOptionValues(APP_STATE_CMD);
                if (states != null) {
                    for (String state : states) {
                        if (!state.trim().isEmpty()) {
                            if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) {
                                allAppStates = true;
                                break;
                            }
                            try {
                                appStates.add(YarnApplicationState.valueOf(StringUtils.toUpperCase(state).trim()));
                            } catch (IllegalArgumentException ex) {
                                sysout.println("The application state " + state + " is invalid.");
                                sysout.println(getAllValidApplicationStates());
                                return exitCode;
                            }
                        }
                    }
                }
            }
            Set<String> appTags = new HashSet<String>();
            if (cliParser.hasOption(APP_TAG_CMD)) {
                String[] tags = cliParser.getOptionValues(APP_TAG_CMD);
                if (tags != null) {
                    for (String tag : tags) {
                        if (!tag.trim().isEmpty()) {
                            appTags.add(tag.trim());
                        }
                    }
                }
            }
            listApplications(appTypes, appStates, appTags);
        } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
            if (args.length != 3) {
                printUsage(title, opts);
                return exitCode;
            }
            listApplicationAttempts(cliParser.getOptionValue(LIST_CMD));
        } else if (args[0].equalsIgnoreCase(CONTAINER)) {
            if (args.length != 3) {
                printUsage(title, opts);
                return exitCode;
            }
            listContainers(cliParser.getOptionValue(LIST_CMD));
        }
    } else if (cliParser.hasOption(KILL_CMD)) {
        if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
            printUsage(title, opts);
            return exitCode;
        }
        return killApplication(cliParser.getOptionValues(KILL_CMD));
    } else if (cliParser.hasOption(MOVE_TO_QUEUE_CMD)) {
        if (!cliParser.hasOption(QUEUE_CMD)) {
            printUsage(title, opts);
            return exitCode;
        }
        moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD), cliParser.getOptionValue(QUEUE_CMD));
    } else if (cliParser.hasOption(FAIL_CMD)) {
        if (!args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
            printUsage(title, opts);
            return exitCode;
        }
        failApplicationAttempt(cliParser.getOptionValue(FAIL_CMD));
    } else if (cliParser.hasOption(HELP_CMD)) {
        printUsage(title, opts);
        return 0;
    } else if (cliParser.hasOption(UPDATE_PRIORITY)) {
        if (!cliParser.hasOption(APP_ID)) {
            printUsage(title, opts);
            return exitCode;
        }
        updateApplicationPriority(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(UPDATE_PRIORITY));
    } else if (cliParser.hasOption(UPDATE_LIFETIME)) {
        if (!cliParser.hasOption(APP_ID)) {
            printUsage(title, opts);
            return exitCode;
        }
        long timeoutInSec = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
        updateApplicationTimeout(cliParser.getOptionValue(APP_ID), ApplicationTimeoutType.LIFETIME, timeoutInSec);
    } else if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
        if (!cliParser.hasOption(APP_ID)) {
            printUsage(title, opts);
            return exitCode;
        }
        moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE));
    } else if (cliParser.hasOption(SIGNAL_CMD)) {
        if (args.length < 3 || args.length > 4) {
            printUsage(title, opts);
            return exitCode;
        }
        final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
        final String containerId = signalArgs[0];
        SignalContainerCommand command = SignalContainerCommand.OUTPUT_THREAD_DUMP;
        if (signalArgs.length == 2) {
            command = SignalContainerCommand.valueOf(signalArgs[1]);
        }
        signalToContainer(containerId, command);
    } else {
        syserr.println("Invalid Command Usage : ");
        printUsage(title, opts);
    }
    return 0;
}
Also used : Options(org.apache.commons.cli.Options) HashSet(java.util.HashSet) EnumSet(java.util.EnumSet) Set(java.util.Set) MissingArgumentException(org.apache.commons.cli.MissingArgumentException) EnumSet(java.util.EnumSet) GnuParser(org.apache.commons.cli.GnuParser) YarnApplicationState(org.apache.hadoop.yarn.api.records.YarnApplicationState) CommandLine(org.apache.commons.cli.CommandLine) SignalContainerCommand(org.apache.hadoop.yarn.api.records.SignalContainerCommand) Option(org.apache.commons.cli.Option)

Example 29 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class UnmanagedAMLauncher method run.

public boolean run() throws IOException, YarnException {
    LOG.info("Starting Client");
    // Connect to ResourceManager
    rmClient.start();
    try {
        // Create launch context for app master
        LOG.info("Setting up application submission context for ASM");
        ApplicationSubmissionContext appContext = rmClient.createApplication().getApplicationSubmissionContext();
        ApplicationId appId = appContext.getApplicationId();
        // set the application name
        appContext.setApplicationName(appName);
        // Set the priority for the application master
        Priority pri = Records.newRecord(Priority.class);
        pri.setPriority(amPriority);
        appContext.setPriority(pri);
        // Set the queue to which this application is to be submitted in the RM
        appContext.setQueue(amQueue);
        // Set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        appContext.setAMContainerSpec(amContainer);
        // unmanaged AM
        appContext.setUnmanagedAM(true);
        LOG.info("Setting unmanaged AM");
        // Submit the application to the applications manager
        LOG.info("Submitting application to ASM");
        rmClient.submitApplication(appContext);
        ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
        if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
            // Monitor the application attempt to wait for launch state
            ApplicationAttemptReport attemptReport = monitorCurrentAppAttempt(appId, YarnApplicationAttemptState.LAUNCHED);
            ApplicationAttemptId attemptId = attemptReport.getApplicationAttemptId();
            LOG.info("Launching AM with application attempt id " + attemptId);
            // launch AM
            launchAM(attemptId);
            // Monitor the application for end state
            appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
        }
        YarnApplicationState appState = appReport.getYarnApplicationState();
        FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();
        LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus);
        boolean success;
        if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) {
            LOG.info("Application has completed successfully.");
            success = true;
        } else {
            LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString() + ", FinalStatus=" + appStatus.toString());
            success = false;
        }
        return success;
    } finally {
        rmClient.stop();
    }
}
Also used : ApplicationReport(org.apache.hadoop.yarn.api.records.ApplicationReport) ApplicationAttemptReport(org.apache.hadoop.yarn.api.records.ApplicationAttemptReport) FinalApplicationStatus(org.apache.hadoop.yarn.api.records.FinalApplicationStatus) Priority(org.apache.hadoop.yarn.api.records.Priority) ApplicationSubmissionContext(org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext) YarnApplicationState(org.apache.hadoop.yarn.api.records.YarnApplicationState) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId)

Example 30 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class UnmanagedAMLauncher method monitorApplication.

/**
   * Monitor the submitted application for completion. Kill application if time
   * expires.
   * 
   * @param appId
   *          Application Id of application to be monitored
   * @return true if application completed successfully
   * @throws YarnException
   * @throws IOException
   */
private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState) throws YarnException, IOException {
    long foundAMCompletedTime = 0;
    StringBuilder expectedFinalState = new StringBuilder();
    boolean first = true;
    for (YarnApplicationState state : finalState) {
        if (first) {
            first = false;
            expectedFinalState.append(state.name());
        } else {
            expectedFinalState.append("," + state.name());
        }
    }
    while (true) {
        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }
        // Get application report for the appId we are interested in
        ApplicationReport report = rmClient.getApplicationReport(appId);
        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId=" + report.getCurrentApplicationAttemptId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }
        // come back
        if (amCompleted) {
            if (foundAMCompletedTime == 0) {
                foundAMCompletedTime = System.currentTimeMillis();
            } else if ((System.currentTimeMillis() - foundAMCompletedTime) > AM_STATE_WAIT_TIMEOUT_MS) {
                LOG.warn("Waited " + AM_STATE_WAIT_TIMEOUT_MS / 1000 + " seconds after process completed for AppReport" + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state + ", ExpectedStates = " + expectedFinalState.toString());
                throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport" + ", CurrentState=" + state + ", ExpectedStates=" + expectedFinalState.toString());
            }
        }
    }
}
Also used : ApplicationReport(org.apache.hadoop.yarn.api.records.ApplicationReport) YarnApplicationState(org.apache.hadoop.yarn.api.records.YarnApplicationState)

Aggregations

YarnApplicationState (org.apache.hadoop.yarn.api.records.YarnApplicationState)41 ApplicationReport (org.apache.hadoop.yarn.api.records.ApplicationReport)27 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)14 YarnException (org.apache.hadoop.yarn.exceptions.YarnException)11 IOException (java.io.IOException)10 FinalApplicationStatus (org.apache.hadoop.yarn.api.records.FinalApplicationStatus)8 HashSet (java.util.HashSet)7 ApplicationSubmissionContext (org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext)7 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)6 ArrayList (java.util.ArrayList)5 ContainerLaunchContext (org.apache.hadoop.yarn.api.records.ContainerLaunchContext)5 YarnClient (org.apache.hadoop.yarn.client.api.YarnClient)5 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)5 Test (org.junit.Test)5 Path (org.apache.hadoop.fs.Path)4 Resource (org.apache.hadoop.yarn.api.records.Resource)4 YarnClientApplication (org.apache.hadoop.yarn.client.api.YarnClientApplication)4 File (java.io.File)3 HashMap (java.util.HashMap)3 FileSystem (org.apache.hadoop.fs.FileSystem)3