Examples with ApplicationReport org.apache.hadoop.yarn.api.records.ApplicationReport used on opensource projects

Example 96 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestYARNRunner method testJobSubmissionFailure.

@Test(timeout = 20000)
public void testJobSubmissionFailure() throws Exception {
    when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class))).thenReturn(appId);
    ApplicationReport report = mock(ApplicationReport.class);
    when(report.getApplicationId()).thenReturn(appId);
    when(report.getDiagnostics()).thenReturn(failString);
    when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
    when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
    Credentials credentials = new Credentials();
    File jobxml = new File(testWorkDir, "job.xml");
    OutputStream out = new FileOutputStream(jobxml);
    conf.writeXml(out);
    out.close();
    try {
        yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
    } catch (IOException io) {
        LOG.info("Logging exception:", io);
        assertTrue(io.getLocalizedMessage().contains(failString));
    }
}

Example 97 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class YarnClientImpl method submitApplication.

@Override
public ApplicationId submitApplication(ApplicationSubmissionContext appContext) throws YarnException, IOException {
    ApplicationId applicationId = appContext.getApplicationId();
    if (applicationId == null) {
        throw new ApplicationIdNotProvidedException("ApplicationId is not provided in ApplicationSubmissionContext");
    }
    SubmitApplicationRequest request = Records.newRecord(SubmitApplicationRequest.class);
    request.setApplicationSubmissionContext(appContext);
    // Only when the security and the timeline service are both enabled
    if (isSecurityEnabled() && timelineV1ServiceEnabled) {
        addTimelineDelegationToken(appContext.getAMContainerSpec());
    }
    //TODO: YARN-1763:Handle RM failovers during the submitApplication call.
    rmClient.submitApplication(request);
    int pollCount = 0;
    long startTime = System.currentTimeMillis();
    EnumSet<YarnApplicationState> waitingStates = EnumSet.of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.SUBMITTED);
    EnumSet<YarnApplicationState> failToSubmitStates = EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.KILLED);
    while (true) {
        try {
            ApplicationReport appReport = getApplicationReport(applicationId);
            YarnApplicationState state = appReport.getYarnApplicationState();
            if (!waitingStates.contains(state)) {
                if (failToSubmitStates.contains(state)) {
                    throw new YarnException("Failed to submit " + applicationId + " to YARN : " + appReport.getDiagnostics());
                }
                LOG.info("Submitted application " + applicationId);
                break;
            }
            long elapsedMillis = System.currentTimeMillis() - startTime;
            if (enforceAsyncAPITimeout() && elapsedMillis >= asyncApiPollTimeoutMillis) {
                throw new YarnException("Timed out while waiting for application " + applicationId + " to be submitted successfully");
            }
            // is blocked here too long.
            if (++pollCount % 10 == 0) {
                LOG.info("Application submission is not finished, " + "submitted application " + applicationId + " is still in " + state);
            }
            try {
                Thread.sleep(submitPollIntervalMillis);
            } catch (InterruptedException ie) {
                String msg = "Interrupted while waiting for application " + applicationId + " to be successfully submitted.";
                LOG.error(msg);
                throw new YarnException(msg, ie);
            }
        } catch (ApplicationNotFoundException ex) {
            // FailOver or RM restart happens before RMStateStore saves
            // ApplicationState
            LOG.info("Re-submit application " + applicationId + "with the " + "same ApplicationSubmissionContext");
            rmClient.submitApplication(request);
        }
    }
    return applicationId;
}

Example 98 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class LogsCLI method runCommand.

private int runCommand(String[] args) throws Exception {
    Options opts = createCommandOpts();
    Options printOpts = createPrintOpts(opts);
    if (args.length < 1) {
        printHelpMessage(printOpts);
        return -1;
    }
    if (args[0].equals("-help")) {
        printHelpMessage(printOpts);
        return 0;
    }
    CommandLineParser parser = new GnuParser();
    String appIdStr = null;
    String containerIdStr = null;
    String nodeAddress = null;
    String appOwner = null;
    boolean getAMContainerLogs = false;
    boolean nodesList = false;
    boolean showApplicationLogInfo = false;
    boolean showContainerLogInfo = false;
    boolean useRegex = false;
    String[] logFiles = null;
    String[] logFilesRegex = null;
    List<String> amContainersList = new ArrayList<String>();
    String localDir = null;
    long bytes = Long.MAX_VALUE;
    try {
        CommandLine commandLine = parser.parse(opts, args, false);
        appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION);
        containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION);
        nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
        appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
        getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
        nodesList = commandLine.hasOption(LIST_NODES_OPTION);
        localDir = commandLine.getOptionValue(OUT_OPTION);
        showApplicationLogInfo = commandLine.hasOption(SHOW_APPLICATION_LOG_INFO);
        showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
        if (getAMContainerLogs) {
            try {
                amContainersList = parseAMContainer(commandLine, printOpts);
            } catch (NumberFormatException ex) {
                System.err.println(ex.getMessage());
                return -1;
            }
        }
        if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) {
            logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION);
        }
        if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_REGEX_OPTION)) {
            logFilesRegex = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_REGEX_OPTION);
            useRegex = true;
        }
        if (commandLine.hasOption(SIZE_OPTION)) {
            bytes = Long.parseLong(commandLine.getOptionValue(SIZE_OPTION));
        }
    } catch (ParseException e) {
        System.err.println("options parsing failed: " + e.getMessage());
        printHelpMessage(printOpts);
        return -1;
    }
    if (appIdStr == null && containerIdStr == null) {
        System.err.println("Both applicationId and containerId are missing, " + " one of them must be specified.");
        printHelpMessage(printOpts);
        return -1;
    }
    ApplicationId appId = null;
    if (appIdStr != null) {
        try {
            appId = ApplicationId.fromString(appIdStr);
        } catch (Exception e) {
            System.err.println("Invalid ApplicationId specified");
            return -1;
        }
    }
    if (containerIdStr != null) {
        try {
            ContainerId containerId = ContainerId.fromString(containerIdStr);
            if (appId == null) {
                appId = containerId.getApplicationAttemptId().getApplicationId();
            } else if (!containerId.getApplicationAttemptId().getApplicationId().equals(appId)) {
                System.err.println("The Application:" + appId + " does not have the container:" + containerId);
                return -1;
            }
        } catch (Exception e) {
            System.err.println("Invalid ContainerId specified");
            return -1;
        }
    }
    if (showApplicationLogInfo && showContainerLogInfo) {
        System.err.println("Invalid options. Can only accept one of " + "show_application_log_info/show_container_log_info.");
        return -1;
    }
    if (logFiles != null && logFiles.length > 0 && logFilesRegex != null && logFilesRegex.length > 0) {
        System.err.println("Invalid options. Can only accept one of " + "log_files/log_files_pattern.");
        return -1;
    }
    if (localDir != null) {
        File file = new File(localDir);
        if (file.exists() && file.isFile()) {
            System.err.println("Invalid value for -out option. " + "Please provide a directory.");
            return -1;
        }
    }
    LogCLIHelpers logCliHelper = new LogCLIHelpers();
    logCliHelper.setConf(getConf());
    YarnApplicationState appState = YarnApplicationState.NEW;
    ApplicationReport appReport = null;
    try {
        appReport = getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
        if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) {
            System.err.println("Logs are not available right now.");
            return -1;
        }
    } catch (IOException | YarnException e) {
        // If we can not get appReport from either RM or ATS
        // We will assume that this app has already finished.
        appState = YarnApplicationState.FINISHED;
        System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem.");
    }
    if (appOwner == null || appOwner.isEmpty()) {
        appOwner = guessAppOwner(appReport, appId);
        if (appOwner == null) {
            System.err.println("Can not find the appOwner. " + "Please specify the correct appOwner");
            System.err.println("Could not locate application logs for " + appId);
            return -1;
        }
    }
    Set<String> logs = new HashSet<String>();
    if (fetchAllLogFiles(logFiles, logFilesRegex)) {
        logs.add("ALL");
    } else if (logFiles != null && logFiles.length > 0) {
        logs.addAll(Arrays.asList(logFiles));
    } else if (logFilesRegex != null && logFilesRegex.length > 0) {
        logs.addAll(Arrays.asList(logFilesRegex));
    }
    ContainerLogsRequest request = new ContainerLogsRequest(appId, isApplicationFinished(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes, null);
    if (showContainerLogInfo) {
        return showContainerLogInfo(request, logCliHelper);
    }
    if (nodesList) {
        return showNodeLists(request, logCliHelper);
    }
    if (showApplicationLogInfo) {
        return showApplicationLogInfo(request, logCliHelper);
    }
    // To get am logs
    if (getAMContainerLogs) {
        return fetchAMContainerLogs(request, amContainersList, logCliHelper, useRegex);
    }
    int resultCode = 0;
    if (containerIdStr != null) {
        return fetchContainerLogs(request, logCliHelper, useRegex);
    } else {
        if (nodeAddress == null) {
            resultCode = fetchApplicationLogs(request, logCliHelper, useRegex);
        } else {
            System.err.println("Should at least provide ContainerId!");
            printHelpMessage(printOpts);
            resultCode = -1;
        }
    }
    return resultCode;
}

Example 99 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class ApplicationCLI method listApplications.

/**
   * Lists the applications matching the given application Types, application
   * States and application Tags present in the Resource Manager.
   * 
   * @param appTypes
   * @param appStates
   * @param appTags
   * @throws YarnException
   * @throws IOException
   */
private void listApplications(Set<String> appTypes, EnumSet<YarnApplicationState> appStates, Set<String> appTags) throws YarnException, IOException {
    PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
    if (allAppStates) {
        for (YarnApplicationState appState : YarnApplicationState.values()) {
            appStates.add(appState);
        }
    } else {
        if (appStates.isEmpty()) {
            appStates.add(YarnApplicationState.RUNNING);
            appStates.add(YarnApplicationState.ACCEPTED);
            appStates.add(YarnApplicationState.SUBMITTED);
        }
    }
    List<ApplicationReport> appsReport = client.getApplications(appTypes, appStates, appTags);
    writer.println("Total number of applications (application-types: " + appTypes + ", states: " + appStates + " and tags: " + appTags + ")" + ":" + appsReport.size());
    writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name", "Application-Type", "User", "Queue", "State", "Final-State", "Progress", "Tracking-URL");
    for (ApplicationReport appReport : appsReport) {
        DecimalFormat formatter = new DecimalFormat("###.##%");
        String progress = formatter.format(appReport.getProgress());
        writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), appReport.getName(), appReport.getApplicationType(), appReport.getUser(), appReport.getQueue(), appReport.getYarnApplicationState(), appReport.getFinalApplicationStatus(), progress, appReport.getOriginalTrackingUrl());
    }
    writer.flush();
}

Example 100 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class ApplicationCLI method moveApplicationAcrossQueues.

/**
   * Moves the application with the given ID to the given queue.
   */
private void moveApplicationAcrossQueues(String applicationId, String queue) throws YarnException, IOException {
    ApplicationId appId = ApplicationId.fromString(applicationId);
    ApplicationReport appReport = client.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED || appReport.getYarnApplicationState() == YarnApplicationState.KILLED || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
        sysout.println("Application " + applicationId + " has already finished ");
    } else {
        sysout.println("Moving application " + applicationId + " to queue " + queue);
        client.moveApplicationAcrossQueues(appId, queue);
        sysout.println("Successfully completed move.");
    }
}