Examples with YarnApplicationState org.apache.hadoop.yarn.api.records.YarnApplicationState used on opensource projects

Example 31 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class ApplicationCLI method listApplications.

/**
   * Lists the applications matching the given application Types, application
   * States and application Tags present in the Resource Manager.
   * 
   * @param appTypes
   * @param appStates
   * @param appTags
   * @throws YarnException
   * @throws IOException
   */
private void listApplications(Set<String> appTypes, EnumSet<YarnApplicationState> appStates, Set<String> appTags) throws YarnException, IOException {
    PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
    if (allAppStates) {
        for (YarnApplicationState appState : YarnApplicationState.values()) {
            appStates.add(appState);
        }
    } else {
        if (appStates.isEmpty()) {
            appStates.add(YarnApplicationState.RUNNING);
            appStates.add(YarnApplicationState.ACCEPTED);
            appStates.add(YarnApplicationState.SUBMITTED);
        }
    }
    List<ApplicationReport> appsReport = client.getApplications(appTypes, appStates, appTags);
    writer.println("Total number of applications (application-types: " + appTypes + ", states: " + appStates + " and tags: " + appTags + ")" + ":" + appsReport.size());
    writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name", "Application-Type", "User", "Queue", "State", "Final-State", "Progress", "Tracking-URL");
    for (ApplicationReport appReport : appsReport) {
        DecimalFormat formatter = new DecimalFormat("###.##%");
        String progress = formatter.format(appReport.getProgress());
        writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), appReport.getName(), appReport.getApplicationType(), appReport.getUser(), appReport.getQueue(), appReport.getYarnApplicationState(), appReport.getFinalApplicationStatus(), progress, appReport.getOriginalTrackingUrl());
    }
    writer.flush();
}

Example 32 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class ApplicationCLI method run.

@Override
public int run(String[] args) throws Exception {
    Options opts = new Options();
    String title = null;
    if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION)) {
        title = APPLICATION;
        opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
        opts.addOption(LIST_CMD, false, "List applications. " + "Supports optional use of -appTypes to filter applications " + "based on application type, -appStates to filter applications " + "based on application state and -appTags to filter applications " + "based on application tag.");
        opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a " + "different queue. Deprecated command. Use 'changeQueue' instead.");
        opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to" + " specify which queue to move an application to.");
        opts.addOption(HELP_CMD, false, "Displays help for all commands.");
        Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " + "filter applications based on " + "input comma-separated list of application types.");
        appTypeOpt.setValueSeparator(',');
        appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
        appTypeOpt.setArgName("Types");
        opts.addOption(appTypeOpt);
        Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " + "to filter applications based on input comma-separated list of " + "application states. " + getAllValidApplicationStates());
        appStateOpt.setValueSeparator(',');
        appStateOpt.setArgs(Option.UNLIMITED_VALUES);
        appStateOpt.setArgName("States");
        opts.addOption(appStateOpt);
        Option appTagOpt = new Option(APP_TAG_CMD, true, "Works with -list to " + "filter applications based on input comma-separated list of " + "application tags.");
        appTagOpt.setValueSeparator(',');
        appTagOpt.setArgs(Option.UNLIMITED_VALUES);
        appTagOpt.setArgName("Tags");
        opts.addOption(appTagOpt);
        opts.addOption(APP_ID, true, "Specify Application Id to be operated");
        opts.addOption(UPDATE_PRIORITY, true, "update priority of an application. ApplicationId can be" + " passed using 'appId' option.");
        opts.addOption(UPDATE_LIFETIME, true, "update timeout of an application from NOW. ApplicationId can be" + " passed using 'appId' option. Timeout value is in seconds.");
        opts.addOption(CHANGE_APPLICATION_QUEUE, true, "Moves application to a new queue. ApplicationId can be" + " passed using 'appId' option. 'movetoqueue' command is" + " deprecated, this new command 'changeQueue' performs same" + " functionality.");
        Option killOpt = new Option(KILL_CMD, true, "Kills the application. " + "Set of applications can be provided separated with space");
        killOpt.setValueSeparator(' ');
        killOpt.setArgs(Option.UNLIMITED_VALUES);
        killOpt.setArgName("Application ID");
        opts.addOption(killOpt);
        opts.getOption(MOVE_TO_QUEUE_CMD).setArgName("Application ID");
        opts.getOption(QUEUE_CMD).setArgName("Queue Name");
        opts.getOption(STATUS_CMD).setArgName("Application ID");
        opts.getOption(APP_ID).setArgName("Application ID");
        opts.getOption(UPDATE_PRIORITY).setArgName("Priority");
        opts.getOption(UPDATE_LIFETIME).setArgName("Timeout");
        opts.getOption(CHANGE_APPLICATION_QUEUE).setArgName("Queue Name");
    } else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
        title = APPLICATION_ATTEMPT;
        opts.addOption(STATUS_CMD, true, "Prints the status of the application attempt.");
        opts.addOption(LIST_CMD, true, "List application attempts for application.");
        opts.addOption(FAIL_CMD, true, "Fails application attempt.");
        opts.addOption(HELP_CMD, false, "Displays help for all commands.");
        opts.getOption(STATUS_CMD).setArgName("Application Attempt ID");
        opts.getOption(LIST_CMD).setArgName("Application ID");
        opts.getOption(FAIL_CMD).setArgName("Application Attempt ID");
    } else if (args.length > 0 && args[0].equalsIgnoreCase(CONTAINER)) {
        title = CONTAINER;
        opts.addOption(STATUS_CMD, true, "Prints the status of the container.");
        opts.addOption(LIST_CMD, true, "List containers for application attempt.");
        opts.addOption(HELP_CMD, false, "Displays help for all commands.");
        opts.getOption(STATUS_CMD).setArgName("Container ID");
        opts.getOption(LIST_CMD).setArgName("Application Attempt ID");
        opts.addOption(SIGNAL_CMD, true, "Signal the container. The available signal commands are " + java.util.Arrays.asList(SignalContainerCommand.values()) + " Default command is OUTPUT_THREAD_DUMP.");
        opts.getOption(SIGNAL_CMD).setArgName("container ID [signal command]");
        opts.getOption(SIGNAL_CMD).setArgs(3);
    }
    int exitCode = -1;
    CommandLine cliParser = null;
    try {
        cliParser = new GnuParser().parse(opts, args);
    } catch (MissingArgumentException ex) {
        sysout.println("Missing argument for options");
        printUsage(title, opts);
        return exitCode;
    }
    if (cliParser.hasOption(STATUS_CMD)) {
        if (args.length != 3) {
            printUsage(title, opts);
            return exitCode;
        }
        if (args[0].equalsIgnoreCase(APPLICATION)) {
            exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
        } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
            exitCode = printApplicationAttemptReport(cliParser.getOptionValue(STATUS_CMD));
        } else if (args[0].equalsIgnoreCase(CONTAINER)) {
            exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD));
        }
        return exitCode;
    } else if (cliParser.hasOption(LIST_CMD)) {
        if (args[0].equalsIgnoreCase(APPLICATION)) {
            allAppStates = false;
            Set<String> appTypes = new HashSet<String>();
            if (cliParser.hasOption(APP_TYPE_CMD)) {
                String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
                if (types != null) {
                    for (String type : types) {
                        if (!type.trim().isEmpty()) {
                            appTypes.add(StringUtils.toUpperCase(type).trim());
                        }
                    }
                }
            }
            EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);
            if (cliParser.hasOption(APP_STATE_CMD)) {
                String[] states = cliParser.getOptionValues(APP_STATE_CMD);
                if (states != null) {
                    for (String state : states) {
                        if (!state.trim().isEmpty()) {
                            if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) {
                                allAppStates = true;
                                break;
                            }
                            try {
                                appStates.add(YarnApplicationState.valueOf(StringUtils.toUpperCase(state).trim()));
                            } catch (IllegalArgumentException ex) {
                                sysout.println("The application state " + state + " is invalid.");
                                sysout.println(getAllValidApplicationStates());
                                return exitCode;
                            }
                        }
                    }
                }
            }
            Set<String> appTags = new HashSet<String>();
            if (cliParser.hasOption(APP_TAG_CMD)) {
                String[] tags = cliParser.getOptionValues(APP_TAG_CMD);
                if (tags != null) {
                    for (String tag : tags) {
                        if (!tag.trim().isEmpty()) {
                            appTags.add(tag.trim());
                        }
                    }
                }
            }
            listApplications(appTypes, appStates, appTags);
        } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
            if (args.length != 3) {
                printUsage(title, opts);
                return exitCode;
            }
            listApplicationAttempts(cliParser.getOptionValue(LIST_CMD));
        } else if (args[0].equalsIgnoreCase(CONTAINER)) {
            if (args.length != 3) {
                printUsage(title, opts);
                return exitCode;
            }
            listContainers(cliParser.getOptionValue(LIST_CMD));
        }
    } else if (cliParser.hasOption(KILL_CMD)) {
        if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
            printUsage(title, opts);
            return exitCode;
        }
        return killApplication(cliParser.getOptionValues(KILL_CMD));
    } else if (cliParser.hasOption(MOVE_TO_QUEUE_CMD)) {
        if (!cliParser.hasOption(QUEUE_CMD)) {
            printUsage(title, opts);
            return exitCode;
        }
        moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD), cliParser.getOptionValue(QUEUE_CMD));
    } else if (cliParser.hasOption(FAIL_CMD)) {
        if (!args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
            printUsage(title, opts);
            return exitCode;
        }
        failApplicationAttempt(cliParser.getOptionValue(FAIL_CMD));
    } else if (cliParser.hasOption(HELP_CMD)) {
        printUsage(title, opts);
        return 0;
    } else if (cliParser.hasOption(UPDATE_PRIORITY)) {
        if (!cliParser.hasOption(APP_ID)) {
            printUsage(title, opts);
            return exitCode;
        }
        updateApplicationPriority(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(UPDATE_PRIORITY));
    } else if (cliParser.hasOption(UPDATE_LIFETIME)) {
        if (!cliParser.hasOption(APP_ID)) {
            printUsage(title, opts);
            return exitCode;
        }
        long timeoutInSec = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
        updateApplicationTimeout(cliParser.getOptionValue(APP_ID), ApplicationTimeoutType.LIFETIME, timeoutInSec);
    } else if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
        if (!cliParser.hasOption(APP_ID)) {
            printUsage(title, opts);
            return exitCode;
        }
        moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE));
    } else if (cliParser.hasOption(SIGNAL_CMD)) {
        if (args.length < 3 || args.length > 4) {
            printUsage(title, opts);
            return exitCode;
        }
        final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
        final String containerId = signalArgs[0];
        SignalContainerCommand command = SignalContainerCommand.OUTPUT_THREAD_DUMP;
        if (signalArgs.length == 2) {
            command = SignalContainerCommand.valueOf(signalArgs[1]);
        }
        signalToContainer(containerId, command);
    } else {
        syserr.println("Invalid Command Usage : ");
        printUsage(title, opts);
    }
    return 0;
}

Example 33 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class UnmanagedAMLauncher method run.

public boolean run() throws IOException, YarnException {
    LOG.info("Starting Client");
    // Connect to ResourceManager
    rmClient.start();
    try {
        // Create launch context for app master
        LOG.info("Setting up application submission context for ASM");
        ApplicationSubmissionContext appContext = rmClient.createApplication().getApplicationSubmissionContext();
        ApplicationId appId = appContext.getApplicationId();
        // set the application name
        appContext.setApplicationName(appName);
        // Set the priority for the application master
        Priority pri = Records.newRecord(Priority.class);
        pri.setPriority(amPriority);
        appContext.setPriority(pri);
        // Set the queue to which this application is to be submitted in the RM
        appContext.setQueue(amQueue);
        // Set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        appContext.setAMContainerSpec(amContainer);
        // unmanaged AM
        appContext.setUnmanagedAM(true);
        LOG.info("Setting unmanaged AM");
        // Submit the application to the applications manager
        LOG.info("Submitting application to ASM");
        rmClient.submitApplication(appContext);
        ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
        if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
            // Monitor the application attempt to wait for launch state
            ApplicationAttemptReport attemptReport = monitorCurrentAppAttempt(appId, YarnApplicationAttemptState.LAUNCHED);
            ApplicationAttemptId attemptId = attemptReport.getApplicationAttemptId();
            LOG.info("Launching AM with application attempt id " + attemptId);
            // launch AM
            launchAM(attemptId);
            // Monitor the application for end state
            appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
        }
        YarnApplicationState appState = appReport.getYarnApplicationState();
        FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();
        LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus);
        boolean success;
        if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) {
            LOG.info("Application has completed successfully.");
            success = true;
        } else {
            LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString() + ", FinalStatus=" + appStatus.toString());
            success = false;
        }
        return success;
    } finally {
        rmClient.stop();
    }
}

Example 34 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class UnmanagedAMLauncher method monitorApplication.

/**
   * Monitor the submitted application for completion. Kill application if time
   * expires.
   * 
   * @param appId
   *          Application Id of application to be monitored
   * @return true if application completed successfully
   * @throws YarnException
   * @throws IOException
   */
private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState) throws YarnException, IOException {
    long foundAMCompletedTime = 0;
    StringBuilder expectedFinalState = new StringBuilder();
    boolean first = true;
    for (YarnApplicationState state : finalState) {
        if (first) {
            first = false;
            expectedFinalState.append(state.name());
        } else {
            expectedFinalState.append("," + state.name());
        }
    }
    while (true) {
        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }
        // Get application report for the appId we are interested in
        ApplicationReport report = rmClient.getApplicationReport(appId);
        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId=" + report.getCurrentApplicationAttemptId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }
        // come back
        if (amCompleted) {
            if (foundAMCompletedTime == 0) {
                foundAMCompletedTime = System.currentTimeMillis();
            } else if ((System.currentTimeMillis() - foundAMCompletedTime) > AM_STATE_WAIT_TIMEOUT_MS) {
                LOG.warn("Waited " + AM_STATE_WAIT_TIMEOUT_MS / 1000 + " seconds after process completed for AppReport" + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state + ", ExpectedStates = " + expectedFinalState.toString());
                throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport" + ", CurrentState=" + state + ", ExpectedStates=" + expectedFinalState.toString());
            }
        }
    }
}

Example 35 with YarnApplicationState

use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.

the class TopCLI method fetchAppReports.

protected List<ApplicationReport> fetchAppReports() throws YarnException, IOException {
    List<ApplicationReport> ret;
    EnumSet<YarnApplicationState> states = EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING);
    GetApplicationsRequest req = GetApplicationsRequest.newInstance(types, states);
    req.setQueues(queues);
    req.setUsers(users);
    ret = applicationReportsCache.getIfPresent(req);
    if (ret != null) {
        return ret;
    }
    ret = client.getApplications(queues, users, types, states);
    applicationReportsCache.put(req, ret);
    return ret;
}