Examples with ContainerLogsRequest org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest used on opensource projects

Example 1 with ContainerLogsRequest

use of org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest in project hadoop by apache.

the class LogsCLI method getMatchedLogOptions.

private ContainerLogsRequest getMatchedLogOptions(ContainerLogsRequest request, LogCLIHelpers logCliHelper, boolean useRegex) throws IOException {
    ContainerLogsRequest newOptions = new ContainerLogsRequest(request);
    if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) {
        Set<String> matchedFiles = new HashSet<String>();
        if (!request.getLogTypes().contains("ALL")) {
            Set<String> files = logCliHelper.listContainerLogs(request);
            matchedFiles = getMatchedLogFiles(request, files, useRegex);
            if (matchedFiles.isEmpty()) {
                return null;
            }
        }
        newOptions.setLogTypes(matchedFiles);
    }
    return newOptions;
}

Example 2 with ContainerLogsRequest

use of org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest in project hadoop by apache.

the class LogsCLI method runCommand.

private int runCommand(String[] args) throws Exception {
    Options opts = createCommandOpts();
    Options printOpts = createPrintOpts(opts);
    if (args.length < 1) {
        printHelpMessage(printOpts);
        return -1;
    }
    if (args[0].equals("-help")) {
        printHelpMessage(printOpts);
        return 0;
    }
    CommandLineParser parser = new GnuParser();
    String appIdStr = null;
    String containerIdStr = null;
    String nodeAddress = null;
    String appOwner = null;
    boolean getAMContainerLogs = false;
    boolean nodesList = false;
    boolean showApplicationLogInfo = false;
    boolean showContainerLogInfo = false;
    boolean useRegex = false;
    String[] logFiles = null;
    String[] logFilesRegex = null;
    List<String> amContainersList = new ArrayList<String>();
    String localDir = null;
    long bytes = Long.MAX_VALUE;
    try {
        CommandLine commandLine = parser.parse(opts, args, false);
        appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION);
        containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION);
        nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
        appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
        getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
        nodesList = commandLine.hasOption(LIST_NODES_OPTION);
        localDir = commandLine.getOptionValue(OUT_OPTION);
        showApplicationLogInfo = commandLine.hasOption(SHOW_APPLICATION_LOG_INFO);
        showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
        if (getAMContainerLogs) {
            try {
                amContainersList = parseAMContainer(commandLine, printOpts);
            } catch (NumberFormatException ex) {
                System.err.println(ex.getMessage());
                return -1;
            }
        }
        if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) {
            logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION);
        }
        if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_REGEX_OPTION)) {
            logFilesRegex = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_REGEX_OPTION);
            useRegex = true;
        }
        if (commandLine.hasOption(SIZE_OPTION)) {
            bytes = Long.parseLong(commandLine.getOptionValue(SIZE_OPTION));
        }
    } catch (ParseException e) {
        System.err.println("options parsing failed: " + e.getMessage());
        printHelpMessage(printOpts);
        return -1;
    }
    if (appIdStr == null && containerIdStr == null) {
        System.err.println("Both applicationId and containerId are missing, " + " one of them must be specified.");
        printHelpMessage(printOpts);
        return -1;
    }
    ApplicationId appId = null;
    if (appIdStr != null) {
        try {
            appId = ApplicationId.fromString(appIdStr);
        } catch (Exception e) {
            System.err.println("Invalid ApplicationId specified");
            return -1;
        }
    }
    if (containerIdStr != null) {
        try {
            ContainerId containerId = ContainerId.fromString(containerIdStr);
            if (appId == null) {
                appId = containerId.getApplicationAttemptId().getApplicationId();
            } else if (!containerId.getApplicationAttemptId().getApplicationId().equals(appId)) {
                System.err.println("The Application:" + appId + " does not have the container:" + containerId);
                return -1;
            }
        } catch (Exception e) {
            System.err.println("Invalid ContainerId specified");
            return -1;
        }
    }
    if (showApplicationLogInfo && showContainerLogInfo) {
        System.err.println("Invalid options. Can only accept one of " + "show_application_log_info/show_container_log_info.");
        return -1;
    }
    if (logFiles != null && logFiles.length > 0 && logFilesRegex != null && logFilesRegex.length > 0) {
        System.err.println("Invalid options. Can only accept one of " + "log_files/log_files_pattern.");
        return -1;
    }
    if (localDir != null) {
        File file = new File(localDir);
        if (file.exists() && file.isFile()) {
            System.err.println("Invalid value for -out option. " + "Please provide a directory.");
            return -1;
        }
    }
    LogCLIHelpers logCliHelper = new LogCLIHelpers();
    logCliHelper.setConf(getConf());
    YarnApplicationState appState = YarnApplicationState.NEW;
    ApplicationReport appReport = null;
    try {
        appReport = getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
        if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) {
            System.err.println("Logs are not available right now.");
            return -1;
        }
    } catch (IOException | YarnException e) {
        // If we can not get appReport from either RM or ATS
        // We will assume that this app has already finished.
        appState = YarnApplicationState.FINISHED;
        System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem.");
    }
    if (appOwner == null || appOwner.isEmpty()) {
        appOwner = guessAppOwner(appReport, appId);
        if (appOwner == null) {
            System.err.println("Can not find the appOwner. " + "Please specify the correct appOwner");
            System.err.println("Could not locate application logs for " + appId);
            return -1;
        }
    }
    Set<String> logs = new HashSet<String>();
    if (fetchAllLogFiles(logFiles, logFilesRegex)) {
        logs.add("ALL");
    } else if (logFiles != null && logFiles.length > 0) {
        logs.addAll(Arrays.asList(logFiles));
    } else if (logFilesRegex != null && logFilesRegex.length > 0) {
        logs.addAll(Arrays.asList(logFilesRegex));
    }
    ContainerLogsRequest request = new ContainerLogsRequest(appId, isApplicationFinished(appState), appOwner, nodeAddress, null, containerIdStr, localDir, logs, bytes, null);
    if (showContainerLogInfo) {
        return showContainerLogInfo(request, logCliHelper);
    }
    if (nodesList) {
        return showNodeLists(request, logCliHelper);
    }
    if (showApplicationLogInfo) {
        return showApplicationLogInfo(request, logCliHelper);
    }
    // To get am logs
    if (getAMContainerLogs) {
        return fetchAMContainerLogs(request, amContainersList, logCliHelper, useRegex);
    }
    int resultCode = 0;
    if (containerIdStr != null) {
        return fetchContainerLogs(request, logCliHelper, useRegex);
    } else {
        if (nodeAddress == null) {
            resultCode = fetchApplicationLogs(request, logCliHelper, useRegex);
        } else {
            System.err.println("Should at least provide ContainerId!");
            printHelpMessage(printOpts);
            resultCode = -1;
        }
    }
    return resultCode;
}

Example 3 with ContainerLogsRequest

use of org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest in project hadoop by apache.

the class LogsCLI method getContainersLogRequestForRunningApplication.

private List<ContainerLogsRequest> getContainersLogRequestForRunningApplication(ContainerLogsRequest options) throws YarnException, IOException {
    List<ContainerLogsRequest> newOptionsList = new ArrayList<ContainerLogsRequest>();
    List<ContainerReport> reports = getContainerReportsFromRunningApplication(options);
    for (ContainerReport container : reports) {
        ContainerLogsRequest newOptions = new ContainerLogsRequest(options);
        newOptions.setContainerId(container.getContainerId().toString());
        newOptions.setNodeId(container.getAssignedNode().toString());
        String httpAddress = container.getNodeHttpAddress();
        if (httpAddress != null && !httpAddress.isEmpty()) {
            newOptions.setNodeHttpAddress(httpAddress.replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
        }
        newOptions.setContainerState(container.getContainerState());
        newOptionsList.add(newOptions);
    }
    return newOptionsList;
}

Example 4 with ContainerLogsRequest

use of org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest in project hadoop by apache.

the class LogsCLI method printAMContainerLogs.

private int printAMContainerLogs(Configuration conf, ContainerLogsRequest request, List<String> amContainers, LogCLIHelpers logCliHelper, boolean useRegex) throws Exception {
    List<JSONObject> amContainersList = null;
    List<ContainerLogsRequest> requests = new ArrayList<ContainerLogsRequest>();
    boolean getAMContainerLists = false;
    String appId = request.getAppId().toString();
    StringBuilder errorMessage = new StringBuilder();
    // which includes nodeAddress for the AM Containers.
    try {
        amContainersList = getAMContainerInfoForRMWebService(conf, appId);
        if (amContainersList != null && !amContainersList.isEmpty()) {
            getAMContainerLists = true;
            for (JSONObject amContainer : amContainersList) {
                ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
                amRequest.setContainerId(amContainer.getString("containerId"));
                String httpAddress = amContainer.getString("nodeHttpAddress");
                if (httpAddress != null && !httpAddress.isEmpty()) {
                    amRequest.setNodeHttpAddress(httpAddress);
                }
                amRequest.setNodeId(amContainer.getString("nodeId"));
                requests.add(amRequest);
            }
        }
    } catch (Exception ex) {
        errorMessage.append(ex.getMessage() + "\n");
        if (request.isAppFinished()) {
            if (!conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
                errorMessage.append("Please enable the timeline service " + "and make sure the timeline server is running.");
            } else {
                try {
                    amContainersList = getAMContainerInfoForAHSWebService(conf, appId);
                    if (amContainersList != null && !amContainersList.isEmpty()) {
                        getAMContainerLists = true;
                        for (JSONObject amContainer : amContainersList) {
                            ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
                            amRequest.setContainerId(amContainer.getString("amContainerId"));
                            requests.add(amRequest);
                        }
                    }
                } catch (Exception e) {
                    errorMessage.append(e.getMessage());
                }
            }
        }
    }
    if (!getAMContainerLists) {
        System.err.println("Unable to get AM container informations " + "for the application:" + appId);
        System.err.println(errorMessage);
        System.err.println("Can not get AMContainers logs for " + "the application:" + appId + " with the appOwner:" + request.getAppOwner());
        return -1;
    }
    if (amContainers.contains("ALL")) {
        for (ContainerLogsRequest amRequest : requests) {
            outputAMContainerLogs(amRequest, conf, logCliHelper, useRegex);
        }
        outStream.println();
        outStream.println("Specified ALL for -am option. " + "Printed logs for all am containers.");
    } else {
        for (String amContainer : amContainers) {
            int amContainerId = Integer.parseInt(amContainer.trim());
            if (amContainerId == -1) {
                outputAMContainerLogs(requests.get(requests.size() - 1), conf, logCliHelper, useRegex);
            } else {
                if (amContainerId <= requests.size()) {
                    outputAMContainerLogs(requests.get(amContainerId - 1), conf, logCliHelper, useRegex);
                } else {
                    System.err.println(String.format("ERROR: Specified AM containerId" + " (%s) exceeds the number of AM containers (%s).", amContainerId, requests.size()));
                    return -1;
                }
            }
        }
    }
    return 0;
}

Example 5 with ContainerLogsRequest

use of org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest in project hadoop by apache.

the class LogsCLI method printContainerLogsFromRunningApplication.

@Private
@VisibleForTesting
public int printContainerLogsFromRunningApplication(Configuration conf, ContainerLogsRequest request, LogCLIHelpers logCliHelper, boolean useRegex) throws IOException {
    String containerIdStr = request.getContainerId().toString();
    String localDir = request.getOutputLocalDir();
    String nodeHttpAddress = request.getNodeHttpAddress();
    if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
        System.err.println("Can not get the logs for the container: " + containerIdStr);
        System.err.println("The node http address is required to get container " + "logs for the Running application.");
        return -1;
    }
    String nodeId = request.getNodeId();
    PrintStream out = logCliHelper.createPrintStream(localDir, nodeId, containerIdStr);
    try {
        Set<String> matchedFiles = getMatchedContainerLogFiles(request, useRegex);
        if (matchedFiles.isEmpty()) {
            System.err.println("Can not find any log file matching the pattern: " + request.getLogTypes() + " for the container: " + containerIdStr + " within the application: " + request.getAppId());
            return -1;
        }
        ContainerLogsRequest newOptions = new ContainerLogsRequest(request);
        newOptions.setLogTypes(matchedFiles);
        Client webServiceClient = Client.create();
        boolean foundAnyLogs = false;
        byte[] buffer = new byte[65536];
        for (String logFile : newOptions.getLogTypes()) {
            InputStream is = null;
            try {
                ClientResponse response = getResponeFromNMWebService(conf, webServiceClient, request, logFile);
                if (response != null && response.getStatusInfo().getStatusCode() == ClientResponse.Status.OK.getStatusCode()) {
                    is = response.getEntityInputStream();
                    int len = 0;
                    while ((len = is.read(buffer)) != -1) {
                        out.write(buffer, 0, len);
                    }
                    out.println();
                } else {
                    out.println("Can not get any logs for the log file: " + logFile);
                    String msg = "Response from the NodeManager:" + nodeId + " WebService is " + ((response == null) ? "null" : "not successful," + " HTTP error code: " + response.getStatus() + ", Server response:\n" + response.getEntity(String.class));
                    out.println(msg);
                }
                out.flush();
                foundAnyLogs = true;
            } catch (ClientHandlerException | UniformInterfaceException ex) {
                System.err.println("Can not find the log file:" + logFile + " for the container:" + containerIdStr + " in NodeManager:" + nodeId);
            } finally {
                IOUtils.closeQuietly(is);
            }
        }
        if (foundAnyLogs) {
            return 0;
        } else {
            return -1;
        }
    } finally {
        logCliHelper.closePrintStream(out);
    }
}