use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.
the class LogsCLI method runCommand.
private int runCommand(String[] args) throws Exception {
Options opts = createCommandOpts();
Options printOpts = createPrintOpts(opts);
if (args.length < 1) {
printHelpMessage(printOpts);
return -1;
}
if (args[0].equals("-help")) {
printHelpMessage(printOpts);
return 0;
}
CommandLineParser parser = new GnuParser();
String appIdStr = null;
String containerIdStr = null;
String nodeAddress = null;
String appOwner = null;
boolean getAMContainerLogs = false;
boolean nodesList = false;
boolean showApplicationLogInfo = false;
boolean showContainerLogInfo = false;
boolean useRegex = false;
String[] logFiles = null;
String[] logFilesRegex = null;
List<String> amContainersList = new ArrayList<String>();
String localDir = null;
long bytes = Long.MAX_VALUE;
try {
CommandLine commandLine = parser.parse(opts, args, false);
appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION);
containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION);
nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
nodesList = commandLine.hasOption(LIST_NODES_OPTION);
localDir = commandLine.getOptionValue(OUT_OPTION);
showApplicationLogInfo = commandLine.hasOption(SHOW_APPLICATION_LOG_INFO);
showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
if (getAMContainerLogs) {
try {
amContainersList = parseAMContainer(commandLine, printOpts);
} catch (NumberFormatException ex) {
System.err.println(ex.getMessage());
return -1;
}
}
if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) {
logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION);
}
if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_REGEX_OPTION)) {
logFilesRegex = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_REGEX_OPTION);
useRegex = true;
}
if (commandLine.hasOption(SIZE_OPTION)) {
bytes = Long.parseLong(commandLine.getOptionValue(SIZE_OPTION));
}
} catch (ParseException e) {
System.err.println("options parsing failed: " + e.getMessage());
printHelpMessage(printOpts);
return -1;
}
if (appIdStr == null && containerIdStr == null) {
System.err.println("Both applicationId and containerId are missing, " + " one of them must be specified.");
printHelpMessage(printOpts);
return -1;
}
ApplicationId appId = null;
if (appIdStr != null) {
try {
appId = ApplicationId.fromString(appIdStr);
} catch (Exception e) {
System.err.println("Invalid ApplicationId specified");
return -1;
}
}
if (containerIdStr != null) {
try {
ContainerId containerId = ContainerId.fromString(containerIdStr);
if (appId == null) {
appId = containerId.getApplicationAttemptId().getApplicationId();
} else if (!containerId.getApplicationAttemptId().getApplicationId().equals(appId)) {
System.err.println("The Application:" + appId + " does not have the container:" + containerId);
return -1;
}
} catch (Exception e) {
System.err.println("Invalid ContainerId specified");
return -1;
}
}
if (showApplicationLogInfo && showContainerLogInfo) {
System.err.println("Invalid options. Can only accept one of " + "show_application_log_info/show_container_log_info.");
return -1;
}
if (logFiles != null && logFiles.length > 0 && logFilesRegex != null && logFilesRegex.length > 0) {
System.err.println("Invalid options. Can only accept one of " + "log_files/log_files_pattern.");
return -1;
}
if (localDir != null) {
File file = new File(localDir);
if (file.exists() && file.isFile()) {
System.err.println("Invalid value for -out option. " + "Please provide a directory.");
return -1;
}
}
LogCLIHelpers logCliHelper = new LogCLIHelpers();
logCliHelper.setConf(getConf());
YarnApplicationState appState = YarnApplicationState.NEW;
ApplicationReport appReport = null;
try {
appReport = getApplicationReport(appId);
appState = appReport.getYarnApplicationState();
if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) {
System.err.println("Logs are not available right now.");
return -1;
}
} catch (IOException | YarnException e) {
// If we can not get appReport from either RM or ATS
// We will assume that this app has already finished.
appState = YarnApplicationState.FINISHED;
System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem.");
}
if (appOwner == null || appOwner.isEmpty()) {
appOwner = guessAppOwner(appReport, appId);
if (appOwner == null) {
System.err.println("Can not find the appOwner. " + "Please specify the correct appOwner");
System.err.println("Could not locate application logs for " + appId);
return -1;
}
}
Set<String> logs = new HashSet<String>();
if (fetchAllLogFiles(logFiles, logFilesRegex)) {
logs.add("ALL");
} else if (logFiles != null && logFiles.length > 0) {
logs.addAll(Arrays.asList(logFiles));
} else if (logFilesRegex != null && logFilesRegex.length > 0) {
logs.addAll(Arrays.asList(logFilesRegex));
}
ContainerLogsRequest request = new ContainerLogsRequest(appId, isApplicationFinished(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes, null);
if (showContainerLogInfo) {
return showContainerLogInfo(request, logCliHelper);
}
if (nodesList) {
return showNodeLists(request, logCliHelper);
}
if (showApplicationLogInfo) {
return showApplicationLogInfo(request, logCliHelper);
}
// To get am logs
if (getAMContainerLogs) {
return fetchAMContainerLogs(request, amContainersList, logCliHelper, useRegex);
}
int resultCode = 0;
if (containerIdStr != null) {
return fetchContainerLogs(request, logCliHelper, useRegex);
} else {
if (nodeAddress == null) {
resultCode = fetchApplicationLogs(request, logCliHelper, useRegex);
} else {
System.err.println("Should at least provide ContainerId!");
printHelpMessage(printOpts);
resultCode = -1;
}
}
return resultCode;
}
use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.
the class ApplicationCLI method listApplications.
/**
* Lists the applications matching the given application Types, application
* States and application Tags present in the Resource Manager.
*
* @param appTypes
* @param appStates
* @param appTags
* @throws YarnException
* @throws IOException
*/
private void listApplications(Set<String> appTypes, EnumSet<YarnApplicationState> appStates, Set<String> appTags) throws YarnException, IOException {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
if (allAppStates) {
for (YarnApplicationState appState : YarnApplicationState.values()) {
appStates.add(appState);
}
} else {
if (appStates.isEmpty()) {
appStates.add(YarnApplicationState.RUNNING);
appStates.add(YarnApplicationState.ACCEPTED);
appStates.add(YarnApplicationState.SUBMITTED);
}
}
List<ApplicationReport> appsReport = client.getApplications(appTypes, appStates, appTags);
writer.println("Total number of applications (application-types: " + appTypes + ", states: " + appStates + " and tags: " + appTags + ")" + ":" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name", "Application-Type", "User", "Queue", "State", "Final-State", "Progress", "Tracking-URL");
for (ApplicationReport appReport : appsReport) {
DecimalFormat formatter = new DecimalFormat("###.##%");
String progress = formatter.format(appReport.getProgress());
writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), appReport.getName(), appReport.getApplicationType(), appReport.getUser(), appReport.getQueue(), appReport.getYarnApplicationState(), appReport.getFinalApplicationStatus(), progress, appReport.getOriginalTrackingUrl());
}
writer.flush();
}
use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.
the class ApplicationCLI method run.
@Override
public int run(String[] args) throws Exception {
Options opts = new Options();
String title = null;
if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION)) {
title = APPLICATION;
opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
opts.addOption(LIST_CMD, false, "List applications. " + "Supports optional use of -appTypes to filter applications " + "based on application type, -appStates to filter applications " + "based on application state and -appTags to filter applications " + "based on application tag.");
opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a " + "different queue. Deprecated command. Use 'changeQueue' instead.");
opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to" + " specify which queue to move an application to.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " + "filter applications based on " + "input comma-separated list of application types.");
appTypeOpt.setValueSeparator(',');
appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
appTypeOpt.setArgName("Types");
opts.addOption(appTypeOpt);
Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " + "to filter applications based on input comma-separated list of " + "application states. " + getAllValidApplicationStates());
appStateOpt.setValueSeparator(',');
appStateOpt.setArgs(Option.UNLIMITED_VALUES);
appStateOpt.setArgName("States");
opts.addOption(appStateOpt);
Option appTagOpt = new Option(APP_TAG_CMD, true, "Works with -list to " + "filter applications based on input comma-separated list of " + "application tags.");
appTagOpt.setValueSeparator(',');
appTagOpt.setArgs(Option.UNLIMITED_VALUES);
appTagOpt.setArgName("Tags");
opts.addOption(appTagOpt);
opts.addOption(APP_ID, true, "Specify Application Id to be operated");
opts.addOption(UPDATE_PRIORITY, true, "update priority of an application. ApplicationId can be" + " passed using 'appId' option.");
opts.addOption(UPDATE_LIFETIME, true, "update timeout of an application from NOW. ApplicationId can be" + " passed using 'appId' option. Timeout value is in seconds.");
opts.addOption(CHANGE_APPLICATION_QUEUE, true, "Moves application to a new queue. ApplicationId can be" + " passed using 'appId' option. 'movetoqueue' command is" + " deprecated, this new command 'changeQueue' performs same" + " functionality.");
Option killOpt = new Option(KILL_CMD, true, "Kills the application. " + "Set of applications can be provided separated with space");
killOpt.setValueSeparator(' ');
killOpt.setArgs(Option.UNLIMITED_VALUES);
killOpt.setArgName("Application ID");
opts.addOption(killOpt);
opts.getOption(MOVE_TO_QUEUE_CMD).setArgName("Application ID");
opts.getOption(QUEUE_CMD).setArgName("Queue Name");
opts.getOption(STATUS_CMD).setArgName("Application ID");
opts.getOption(APP_ID).setArgName("Application ID");
opts.getOption(UPDATE_PRIORITY).setArgName("Priority");
opts.getOption(UPDATE_LIFETIME).setArgName("Timeout");
opts.getOption(CHANGE_APPLICATION_QUEUE).setArgName("Queue Name");
} else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
title = APPLICATION_ATTEMPT;
opts.addOption(STATUS_CMD, true, "Prints the status of the application attempt.");
opts.addOption(LIST_CMD, true, "List application attempts for application.");
opts.addOption(FAIL_CMD, true, "Fails application attempt.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
opts.getOption(STATUS_CMD).setArgName("Application Attempt ID");
opts.getOption(LIST_CMD).setArgName("Application ID");
opts.getOption(FAIL_CMD).setArgName("Application Attempt ID");
} else if (args.length > 0 && args[0].equalsIgnoreCase(CONTAINER)) {
title = CONTAINER;
opts.addOption(STATUS_CMD, true, "Prints the status of the container.");
opts.addOption(LIST_CMD, true, "List containers for application attempt.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
opts.getOption(STATUS_CMD).setArgName("Container ID");
opts.getOption(LIST_CMD).setArgName("Application Attempt ID");
opts.addOption(SIGNAL_CMD, true, "Signal the container. The available signal commands are " + java.util.Arrays.asList(SignalContainerCommand.values()) + " Default command is OUTPUT_THREAD_DUMP.");
opts.getOption(SIGNAL_CMD).setArgName("container ID [signal command]");
opts.getOption(SIGNAL_CMD).setArgs(3);
}
int exitCode = -1;
CommandLine cliParser = null;
try {
cliParser = new GnuParser().parse(opts, args);
} catch (MissingArgumentException ex) {
sysout.println("Missing argument for options");
printUsage(title, opts);
return exitCode;
}
if (cliParser.hasOption(STATUS_CMD)) {
if (args.length != 3) {
printUsage(title, opts);
return exitCode;
}
if (args[0].equalsIgnoreCase(APPLICATION)) {
exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
} else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
exitCode = printApplicationAttemptReport(cliParser.getOptionValue(STATUS_CMD));
} else if (args[0].equalsIgnoreCase(CONTAINER)) {
exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD));
}
return exitCode;
} else if (cliParser.hasOption(LIST_CMD)) {
if (args[0].equalsIgnoreCase(APPLICATION)) {
allAppStates = false;
Set<String> appTypes = new HashSet<String>();
if (cliParser.hasOption(APP_TYPE_CMD)) {
String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
if (types != null) {
for (String type : types) {
if (!type.trim().isEmpty()) {
appTypes.add(StringUtils.toUpperCase(type).trim());
}
}
}
}
EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);
if (cliParser.hasOption(APP_STATE_CMD)) {
String[] states = cliParser.getOptionValues(APP_STATE_CMD);
if (states != null) {
for (String state : states) {
if (!state.trim().isEmpty()) {
if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) {
allAppStates = true;
break;
}
try {
appStates.add(YarnApplicationState.valueOf(StringUtils.toUpperCase(state).trim()));
} catch (IllegalArgumentException ex) {
sysout.println("The application state " + state + " is invalid.");
sysout.println(getAllValidApplicationStates());
return exitCode;
}
}
}
}
}
Set<String> appTags = new HashSet<String>();
if (cliParser.hasOption(APP_TAG_CMD)) {
String[] tags = cliParser.getOptionValues(APP_TAG_CMD);
if (tags != null) {
for (String tag : tags) {
if (!tag.trim().isEmpty()) {
appTags.add(tag.trim());
}
}
}
}
listApplications(appTypes, appStates, appTags);
} else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
if (args.length != 3) {
printUsage(title, opts);
return exitCode;
}
listApplicationAttempts(cliParser.getOptionValue(LIST_CMD));
} else if (args[0].equalsIgnoreCase(CONTAINER)) {
if (args.length != 3) {
printUsage(title, opts);
return exitCode;
}
listContainers(cliParser.getOptionValue(LIST_CMD));
}
} else if (cliParser.hasOption(KILL_CMD)) {
if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
printUsage(title, opts);
return exitCode;
}
return killApplication(cliParser.getOptionValues(KILL_CMD));
} else if (cliParser.hasOption(MOVE_TO_QUEUE_CMD)) {
if (!cliParser.hasOption(QUEUE_CMD)) {
printUsage(title, opts);
return exitCode;
}
moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD), cliParser.getOptionValue(QUEUE_CMD));
} else if (cliParser.hasOption(FAIL_CMD)) {
if (!args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
printUsage(title, opts);
return exitCode;
}
failApplicationAttempt(cliParser.getOptionValue(FAIL_CMD));
} else if (cliParser.hasOption(HELP_CMD)) {
printUsage(title, opts);
return 0;
} else if (cliParser.hasOption(UPDATE_PRIORITY)) {
if (!cliParser.hasOption(APP_ID)) {
printUsage(title, opts);
return exitCode;
}
updateApplicationPriority(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(UPDATE_PRIORITY));
} else if (cliParser.hasOption(UPDATE_LIFETIME)) {
if (!cliParser.hasOption(APP_ID)) {
printUsage(title, opts);
return exitCode;
}
long timeoutInSec = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
updateApplicationTimeout(cliParser.getOptionValue(APP_ID), ApplicationTimeoutType.LIFETIME, timeoutInSec);
} else if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
if (!cliParser.hasOption(APP_ID)) {
printUsage(title, opts);
return exitCode;
}
moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE));
} else if (cliParser.hasOption(SIGNAL_CMD)) {
if (args.length < 3 || args.length > 4) {
printUsage(title, opts);
return exitCode;
}
final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
final String containerId = signalArgs[0];
SignalContainerCommand command = SignalContainerCommand.OUTPUT_THREAD_DUMP;
if (signalArgs.length == 2) {
command = SignalContainerCommand.valueOf(signalArgs[1]);
}
signalToContainer(containerId, command);
} else {
syserr.println("Invalid Command Usage : ");
printUsage(title, opts);
}
return 0;
}
use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.
the class UnmanagedAMLauncher method run.
public boolean run() throws IOException, YarnException {
LOG.info("Starting Client");
// Connect to ResourceManager
rmClient.start();
try {
// Create launch context for app master
LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = rmClient.createApplication().getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
// set the application name
appContext.setApplicationName(appName);
// Set the priority for the application master
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue(amQueue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
appContext.setAMContainerSpec(amContainer);
// unmanaged AM
appContext.setUnmanagedAM(true);
LOG.info("Setting unmanaged AM");
// Submit the application to the applications manager
LOG.info("Submitting application to ASM");
rmClient.submitApplication(appContext);
ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
// Monitor the application attempt to wait for launch state
ApplicationAttemptReport attemptReport = monitorCurrentAppAttempt(appId, YarnApplicationAttemptState.LAUNCHED);
ApplicationAttemptId attemptId = attemptReport.getApplicationAttemptId();
LOG.info("Launching AM with application attempt id " + attemptId);
// launch AM
launchAM(attemptId);
// Monitor the application for end state
appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
}
YarnApplicationState appState = appReport.getYarnApplicationState();
FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();
LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus);
boolean success;
if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) {
LOG.info("Application has completed successfully.");
success = true;
} else {
LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString() + ", FinalStatus=" + appStatus.toString());
success = false;
}
return success;
} finally {
rmClient.stop();
}
}
use of org.apache.hadoop.yarn.api.records.YarnApplicationState in project hadoop by apache.
the class UnmanagedAMLauncher method monitorApplication.
/**
* Monitor the submitted application for completion. Kill application if time
* expires.
*
* @param appId
* Application Id of application to be monitored
* @return true if application completed successfully
* @throws YarnException
* @throws IOException
*/
private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState) throws YarnException, IOException {
long foundAMCompletedTime = 0;
StringBuilder expectedFinalState = new StringBuilder();
boolean first = true;
for (YarnApplicationState state : finalState) {
if (first) {
first = false;
expectedFinalState.append(state.name());
} else {
expectedFinalState.append("," + state.name());
}
}
while (true) {
// Check app status every 1 second.
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
LOG.debug("Thread sleep in monitoring loop interrupted");
}
// Get application report for the appId we are interested in
ApplicationReport report = rmClient.getApplicationReport(appId);
LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId=" + report.getCurrentApplicationAttemptId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
YarnApplicationState state = report.getYarnApplicationState();
if (finalState.contains(state)) {
return report;
}
// come back
if (amCompleted) {
if (foundAMCompletedTime == 0) {
foundAMCompletedTime = System.currentTimeMillis();
} else if ((System.currentTimeMillis() - foundAMCompletedTime) > AM_STATE_WAIT_TIMEOUT_MS) {
LOG.warn("Waited " + AM_STATE_WAIT_TIMEOUT_MS / 1000 + " seconds after process completed for AppReport" + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state + ", ExpectedStates = " + expectedFinalState.toString());
throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport" + ", CurrentState=" + state + ", ExpectedStates=" + expectedFinalState.toString());
}
}
}
}
Aggregations