Examples with YarnClient org.apache.hadoop.yarn.client.api.YarnClient used on opensource projects

Example 16 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method testHelpMessage.

@Test(timeout = 5000l)
public void testHelpMessage() throws Exception {
    Configuration conf = new YarnConfiguration();
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName());
    LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
    dumper.setConf(conf);
    int exitCode = dumper.run(new String[] {});
    assertTrue(exitCode == -1);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter pw = new PrintWriter(baos);
    pw.println("Retrieve logs for YARN applications.");
    pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
    pw.println();
    pw.println("general options are:");
    pw.println(" -am <AM Containers>                     Prints the AM Container logs for");
    pw.println("                                         this application. Specify");
    pw.println("                                         comma-separated value to get logs");
    pw.println("                                         for related AM Container. For");
    pw.println("                                         example, If we specify -am 1,2,");
    pw.println("                                         we will get the logs for the");
    pw.println("                                         first AM Container as well as the");
    pw.println("                                         second AM Container. To get logs");
    pw.println("                                         for all AM Containers, use -am");
    pw.println("                                         ALL. To get logs for the latest");
    pw.println("                                         AM Container, use -am -1. By");
    pw.println("                                         default, it will print all");
    pw.println("                                         available logs. Work with");
    pw.println("                                         -log_files to get only specific");
    pw.println("                                         logs.");
    pw.println(" -appOwner <Application Owner>           AppOwner (assumed to be current");
    pw.println("                                         user if not specified)");
    pw.println(" -containerId <Container ID>             ContainerId. By default, it will");
    pw.println("                                         print all available logs. Work");
    pw.println("                                         with -log_files to get only");
    pw.println("                                         specific logs. If specified, the");
    pw.println("                                         applicationId can be omitted");
    pw.println(" -help                                   Displays help for all commands.");
    pw.println(" -list_nodes                             Show the list of nodes that");
    pw.println("                                         successfully aggregated logs.");
    pw.println("                                         This option can only be used with");
    pw.println("                                         finished applications.");
    pw.println(" -log_files <Log File Name>              Specify comma-separated value to");
    pw.println("                                         get exact matched log files. Use");
    pw.println("                                         \"ALL\" or \"*\" to fetch all the log");
    pw.println("                                         files for the container.");
    pw.println(" -log_files_pattern <Log File Pattern>   Specify comma-separated value to");
    pw.println("                                         get matched log files by using");
    pw.println("                                         java regex. Use \".*\" to fetch all");
    pw.println("                                         the log files for the container.");
    pw.println(" -nodeAddress <Node Address>             NodeAddress in the format");
    pw.println("                                         nodename:port");
    pw.println(" -out <Local Directory>                  Local directory for storing");
    pw.println("                                         individual container logs. The");
    pw.println("                                         container logs will be stored");
    pw.println("                                         based on the node the container");
    pw.println("                                         ran on.");
    pw.println(" -show_application_log_info              Show the containerIds which");
    pw.println("                                         belong to the specific");
    pw.println("                                         Application. You can combine this");
    pw.println("                                         with --nodeAddress to get");
    pw.println("                                         containerIds for all the");
    pw.println("                                         containers on the specific");
    pw.println("                                         NodeManager.");
    pw.println(" -show_container_log_info                Show the container log metadata,");
    pw.println("                                         including log-file names, the");
    pw.println("                                         size of the log files. You can");
    pw.println("                                         combine this with --containerId");
    pw.println("                                         to get log metadata for the");
    pw.println("                                         specific container, or with");
    pw.println("                                         --nodeAddress to get log metadata");
    pw.println("                                         for all the containers on the");
    pw.println("                                         specific NodeManager.");
    pw.println(" -size <size>                            Prints the log file's first 'n'");
    pw.println("                                         bytes or the last 'n' bytes. Use");
    pw.println("                                         negative values as bytes to read");
    pw.println("                                         from the end and positive values");
    pw.println("                                         as bytes to read from the");
    pw.println("                                         beginning.");
    pw.close();
    String appReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appReportStr, sysOutStream.toString());
}

Example 17 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method testFetchFinishedApplictionLogs.

@Test(timeout = 15000)
public void testFetchFinishedApplictionLogs() throws Exception {
    String remoteLogRootDir = "target/logs/";
    Configuration configuration = new Configuration();
    configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
    configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
    FileSystem fs = FileSystem.get(configuration);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0);
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
    ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3);
    final NodeId nodeId = NodeId.newInstance("localhost", 1234);
    // create local logs
    String rootLogDir = "target/LocalLogs";
    Path rootLogDirPath = new Path(rootLogDir);
    if (fs.exists(rootLogDirPath)) {
        fs.delete(rootLogDirPath, true);
    }
    assertTrue(fs.mkdirs(rootLogDirPath));
    Path appLogsDir = new Path(rootLogDirPath, appId.toString());
    if (fs.exists(appLogsDir)) {
        fs.delete(appLogsDir, true);
    }
    assertTrue(fs.mkdirs(appLogsDir));
    List<String> rootLogDirs = Arrays.asList(rootLogDir);
    List<String> logTypes = new ArrayList<String>();
    logTypes.add("syslog");
    // create container logs in localLogDir
    createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes);
    createContainerLogInLocalDir(appLogsDir, containerId2, fs, logTypes);
    // create two logs for container3 in localLogDir
    logTypes.add("stdout");
    logTypes.add("stdout1234");
    createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes);
    Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/logs/application_0_0001");
    if (fs.exists(path)) {
        fs.delete(path, true);
    }
    assertTrue(fs.mkdirs(path));
    // upload container logs into remote directory
    // the first two logs is empty. When we try to read first two logs,
    // we will meet EOF exception, but it will not impact other logs.
    // Other logs should be read successfully.
    uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0, path, fs);
    uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs);
    uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs);
    uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2, path, fs);
    uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId3, path, fs);
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, ugi.getShortUserName());
    LogsCLI cli = new LogsCLIForTest(mockYarnClient) {

        @Override
        public ContainerReport getContainerReport(String containerIdStr) throws YarnException, IOException {
            ContainerReport mockReport = mock(ContainerReport.class);
            doReturn(nodeId).when(mockReport).getAssignedNode();
            doReturn("http://localhost:2345").when(mockReport).getNodeHttpAddress();
            return mockReport;
        }
    };
    cli.setConf(configuration);
    int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files_pattern", ".*" });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "*" });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    int fullSize = sysOutStream.toByteArray().length;
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "stdout" });
    assertTrue(exitCode == 0);
    assertFalse(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files_pattern", "std*" });
    assertTrue(exitCode == 0);
    assertFalse(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
    assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "123" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Can not find any log file matching the pattern: [123] " + "for the application: " + appId.toString()));
    sysErrStream.reset();
    // specify the bytes which is larger than the actual file size,
    // we would get the full logs
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "*", "-size", "10000" });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toByteArray().length == fullSize);
    sysOutStream.reset();
    // uploaded two logs for container1. The first log is empty.
    // The second one is not empty.
    // We can still successfully read logs for container1.
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId1.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
    assertTrue(sysOutStream.toString().contains("Log Upload Time"));
    assertTrue(!sysOutStream.toString().contains("Logs for container " + containerId1.toString() + " are not present in this log-file."));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "123" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Can not find any log file matching the pattern: [123] " + "for the container: " + containerId3 + " within the application: " + appId.toString()));
    sysErrStream.reset();
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout" });
    assertTrue(exitCode == 0);
    int fullContextSize = sysOutStream.toByteArray().length;
    String fullContext = sysOutStream.toString();
    sysOutStream.reset();
    String logMessage = logMessage(containerId3, "stdout");
    int fileContentSize = logMessage.getBytes().length;
    int tailContentSize = "\nEnd of LogType:stdout\n\n".getBytes().length;
    // specify how many bytes we should get from logs
    // specify a position number, it would get the first n bytes from
    // container log
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", "5" });
    assertTrue(exitCode == 0);
    Assert.assertEquals(new String(logMessage.getBytes(), 0, 5), new String(sysOutStream.toByteArray(), (fullContextSize - fileContentSize - tailContentSize), 5));
    sysOutStream.reset();
    // specify a negative number, it would get the last n bytes from
    // container log
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", "-5" });
    assertTrue(exitCode == 0);
    Assert.assertEquals(new String(logMessage.getBytes(), logMessage.getBytes().length - 5, 5), new String(sysOutStream.toByteArray(), (fullContextSize - fileContentSize - tailContentSize), 5));
    sysOutStream.reset();
    long negative = (fullContextSize + 1000) * (-1);
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", Long.toString(negative) });
    assertTrue(exitCode == 0);
    Assert.assertEquals(fullContext, sysOutStream.toString());
    sysOutStream.reset();
    // Uploaded the empty log for container0.
    // We should see the message showing the log for container0
    // are not present.
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId0.toString() });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Logs for container " + containerId0.toString() + " are not present in this log-file."));
    sysErrStream.reset();
    // uploaded two logs for container3. The first log is named as syslog.
    // The second one is named as stdout.
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId3.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    sysOutStream.reset();
    // set -log_files option as stdout
    // should only print log with the name as stdout
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout" });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(!sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    sysOutStream.reset();
    YarnClient mockYarnClientWithException = createMockYarnClientWithException();
    cli = new LogsCLIForTest(mockYarnClientWithException);
    cli.setConf(configuration);
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
    sysOutStream.reset();
    // The same should also work without the applicationId
    exitCode = cli.run(new String[] { "-containerId", containerId3.toString() });
    assertTrue(exitCode == 0);
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
    assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
    assertTrue(sysOutStream.toString().contains(containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
    sysOutStream.reset();
    exitCode = cli.run(new String[] { "-containerId", "invalid_container" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Invalid ContainerId specified"));
    sysErrStream.reset();
    fs.delete(new Path(remoteLogRootDir), true);
    fs.delete(new Path(rootLogDir), true);
}

Example 18 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method createMockYarnClient.

private YarnClient createMockYarnClient(YarnApplicationState appState, String user, boolean mockContainerReport, List<ApplicationAttemptReport> mockAttempts, List<ContainerReport> mockContainers) throws YarnException, IOException {
    YarnClient mockClient = mock(YarnClient.class);
    ApplicationReport mockAppReport = mock(ApplicationReport.class);
    doReturn(user).when(mockAppReport).getUser();
    doReturn(appState).when(mockAppReport).getYarnApplicationState();
    doReturn(mockAppReport).when(mockClient).getApplicationReport(any(ApplicationId.class));
    if (mockContainerReport) {
        doReturn(mockAttempts).when(mockClient).getApplicationAttempts(any(ApplicationId.class));
        doReturn(mockContainers).when(mockClient).getContainers(any(ApplicationAttemptId.class));
    }
    return mockClient;
}

Example 19 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method testLogsCLIWithInvalidArgs.

@Test(timeout = 5000)
public void testLogsCLIWithInvalidArgs() throws Exception {
    String localDir = "target/SaveLogs";
    Path localPath = new Path(localDir);
    Configuration configuration = new Configuration();
    FileSystem fs = FileSystem.get(configuration);
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName());
    LogsCLI cli = new LogsCLIForTest(mockYarnClient);
    cli.setConf(configuration);
    // Specify an invalid applicationId
    int exitCode = cli.run(new String[] { "-applicationId", "123" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Invalid ApplicationId specified"));
    sysErrStream.reset();
    // Specify an invalid containerId
    exitCode = cli.run(new String[] { "-containerId", "123" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Invalid ContainerId specified"));
    sysErrStream.reset();
    // Specify show_container_log_info and show_application_log_info
    // at the same time
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-show_container_log_info", "-show_application_log_info" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Invalid options. " + "Can only accept one of show_application_log_info/" + "show_container_log_info."));
    sysErrStream.reset();
    // Specify log_files and log_files_pattern
    // at the same time
    exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "*", "-log_files_pattern", ".*" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("Invalid options. " + "Can only accept one of log_files/" + "log_files_pattern."));
    sysErrStream.reset();
    // Specify a file name to the option -out
    try {
        fs.mkdirs(localPath);
        Path tmpFilePath = new Path(localPath, "tmpFile");
        if (!fs.exists(tmpFilePath)) {
            fs.createNewFile(tmpFilePath);
        }
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-out", tmpFilePath.toString() });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Invalid value for -out option. Please provide a directory."));
    } finally {
        fs.delete(localPath, true);
    }
}

Example 20 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method testInvalidOpts.

@Test(timeout = 1000l)
public void testInvalidOpts() throws Exception {
    Configuration conf = new YarnConfiguration();
    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName());
    LogsCLI cli = new LogsCLIForTest(mockYarnClient);
    cli.setConf(conf);
    int exitCode = cli.run(new String[] { "-InvalidOpts" });
    assertTrue(exitCode == -1);
    assertTrue(sysErrStream.toString().contains("options parsing failed: Unrecognized option: -InvalidOpts"));
}