Examples with LogKey org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey used on opensource projects

Example 1 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class TestAggregatedLogFormat method testContainerLogsFileAccess.

@Test(timeout = 10000)
public void testContainerLogsFileAccess() throws IOException {
    // This test will run only if NativeIO is enabled as SecureIOUtils 
    // require it to be enabled.
    Assume.assumeTrue(NativeIO.isAvailable());
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    File workDir = new File(testWorkDir, "testContainerLogsFileAccess1");
    Path remoteAppLogFile = new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
    Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
    String data = "Log File content for container : ";
    // Creating files for container1. Log aggregator will try to read log files
    // with illegal user.
    ApplicationId applicationId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ContainerId testContainerId1 = ContainerId.newContainerId(applicationAttemptId, 1);
    Path appDir = new Path(srcFileRoot, testContainerId1.getApplicationAttemptId().getApplicationId().toString());
    Path srcFilePath1 = new Path(appDir, testContainerId1.toString());
    String stdout = "stdout";
    String stderr = "stderr";
    writeSrcFile(srcFilePath1, stdout, data + testContainerId1.toString() + stdout);
    writeSrcFile(srcFilePath1, stderr, data + testContainerId1.toString() + stderr);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);
    LogKey logKey = new LogKey(testContainerId1);
    String randomUser = "randomUser";
    LogValue logValue = spy(new LogValue(Collections.singletonList(srcFileRoot.toString()), testContainerId1, randomUser));
    // It is trying simulate a situation where first log file is owned by
    // different user (probably symlink) and second one by the user itself.
    // The first file should not be aggregated. Because this log file has the invalid
    // user name.
    when(logValue.getUser()).thenReturn(randomUser).thenReturn(ugi.getShortUserName());
    logWriter.append(logKey, logValue);
    logWriter.close();
    BufferedReader in = new BufferedReader(new FileReader(new File(remoteAppLogFile.toUri().getRawPath())));
    String line;
    StringBuffer sb = new StringBuffer("");
    while ((line = in.readLine()) != null) {
        LOG.info(line);
        sb.append(line);
    }
    line = sb.toString();
    String expectedOwner = ugi.getShortUserName();
    if (Path.WINDOWS) {
        final String adminsGroupString = "Administrators";
        if (Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString)) {
            expectedOwner = adminsGroupString;
        }
    }
    // This file: stderr should not be aggregated.
    // And we will not aggregate the log message.
    String stdoutFile1 = StringUtils.join(File.separator, Arrays.asList(new String[] { workDir.getAbsolutePath(), "srcFiles", testContainerId1.getApplicationAttemptId().getApplicationId().toString(), testContainerId1.toString(), stderr }));
    // The file: stdout is expected to be aggregated.
    String stdoutFile2 = StringUtils.join(File.separator, Arrays.asList(new String[] { workDir.getAbsolutePath(), "srcFiles", testContainerId1.getApplicationAttemptId().getApplicationId().toString(), testContainerId1.toString(), stdout }));
    String message2 = "Owner '" + expectedOwner + "' for path " + stdoutFile2 + " did not match expected owner '" + ugi.getShortUserName() + "'";
    Assert.assertFalse(line.contains(message2));
    Assert.assertFalse(line.contains(data + testContainerId1.toString() + stderr));
    Assert.assertTrue(line.contains(data + testContainerId1.toString() + stdout));
}

Example 2 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class TestAggregatedLogFormat method writeSrcFileAndALog.

private void writeSrcFileAndALog(Path srcFilePath, String fileName, final long length, Path remoteAppLogFile, Path srcFileRoot, ContainerId testContainerId) throws Exception {
    File dir = new File(srcFilePath.toString());
    if (!dir.exists()) {
        if (!dir.mkdirs()) {
            throw new IOException("Unable to create directory : " + dir);
        }
    }
    File outputFile = new File(new File(srcFilePath.toString()), fileName);
    FileOutputStream os = new FileOutputStream(outputFile);
    final OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8");
    final int ch = filler;
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    LogWriter logWriter = new LogWriter(new Configuration(), remoteAppLogFile, ugi);
    LogKey logKey = new LogKey(testContainerId);
    LogValue logValue = spy(new LogValue(Collections.singletonList(srcFileRoot.toString()), testContainerId, ugi.getShortUserName()));
    final CountDownLatch latch = new CountDownLatch(1);
    Thread t = new Thread() {

        public void run() {
            try {
                for (int i = 0; i < length / 3; i++) {
                    osw.write(ch);
                }
                latch.countDown();
                for (int i = 0; i < (2 * length) / 3; i++) {
                    osw.write(ch);
                }
                osw.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    };
    t.start();
    //Wait till the osw is partially written
    //aggregation starts once the ows has completed 1/3rd of its work
    latch.await();
    //Aggregate The Logs
    logWriter.append(logKey, logValue);
    logWriter.close();
}

Example 3 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class LogCLIHelpers method dumpAllContainersLogs.

@Private
public int dumpAllContainersLogs(ContainerLogsRequest options) throws IOException {
    ApplicationId appId = options.getAppId();
    String appOwner = options.getAppOwner();
    String localDir = options.getOutputLocalDir();
    List<String> logTypes = new ArrayList<String>(options.getLogTypes());
    RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(appId, appOwner);
    if (nodeFiles == null) {
        return -1;
    }
    boolean foundAnyLogs = false;
    while (nodeFiles.hasNext()) {
        FileStatus thisNodeFile = nodeFiles.next();
        if (thisNodeFile.getPath().getName().equals(appId + ".har")) {
            Path p = new Path("har:///" + thisNodeFile.getPath().toUri().getRawPath());
            nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p);
            continue;
        }
        if (!thisNodeFile.getPath().getName().endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
            AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath());
            try {
                DataInputStream valueStream;
                LogKey key = new LogKey();
                valueStream = reader.next(key);
                while (valueStream != null) {
                    PrintStream out = createPrintStream(localDir, thisNodeFile.getPath().getName(), key.toString());
                    try {
                        String containerString = String.format(CONTAINER_ON_NODE_PATTERN, key, thisNodeFile.getPath().getName());
                        out.println(containerString);
                        out.println("LogAggregationType: AGGREGATED");
                        out.println(StringUtils.repeat("=", containerString.length()));
                        while (true) {
                            try {
                                if (logTypes == null || logTypes.isEmpty()) {
                                    LogReader.readAContainerLogsForALogType(valueStream, out, thisNodeFile.getModificationTime(), options.getBytes());
                                    foundAnyLogs = true;
                                } else {
                                    int result = LogReader.readContainerLogsForALogType(valueStream, out, thisNodeFile.getModificationTime(), logTypes, options.getBytes());
                                    if (result == 0) {
                                        foundAnyLogs = true;
                                    }
                                }
                            } catch (EOFException eof) {
                                break;
                            }
                        }
                    } finally {
                        closePrintStream(out);
                    }
                    // Next container
                    key = new LogKey();
                    valueStream = reader.next(key);
                }
            } finally {
                reader.close();
            }
        }
    }
    if (!foundAnyLogs) {
        emptyLogDir(LogAggregationUtils.getRemoteAppLogDir(conf, appId, appOwner).toString());
        return -1;
    }
    return 0;
}

Example 4 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class LogCLIHelpers method getContainerLogsStream.

private DataInputStream getContainerLogsStream(String containerIdStr, AggregatedLogFormat.LogReader reader) throws IOException {
    DataInputStream valueStream;
    LogKey key = new LogKey();
    valueStream = reader.next(key);
    while (valueStream != null && !key.toString().equals(containerIdStr)) {
        // Next container
        key = new LogKey();
        valueStream = reader.next(key);
    }
    return valueStream;
}

Example 5 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class LogCLIHelpers method printContainersList.

@Private
public void printContainersList(ContainerLogsRequest options, PrintStream out, PrintStream err) throws IOException {
    ApplicationId appId = options.getAppId();
    String appOwner = options.getAppOwner();
    String nodeId = options.getNodeId();
    String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId);
    RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(appId, appOwner);
    if (nodeFiles == null) {
        return;
    }
    boolean foundAnyLogs = false;
    while (nodeFiles.hasNext()) {
        FileStatus thisNodeFile = nodeFiles.next();
        if (nodeIdStr != null) {
            if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
                continue;
            }
        }
        if (!thisNodeFile.getPath().getName().endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
            AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath());
            try {
                DataInputStream valueStream;
                LogKey key = new LogKey();
                valueStream = reader.next(key);
                while (valueStream != null) {
                    out.println(String.format(CONTAINER_ON_NODE_PATTERN, key, thisNodeFile.getPath().getName()));
                    foundAnyLogs = true;
                    // Next container
                    key = new LogKey();
                    valueStream = reader.next(key);
                }
            } finally {
                reader.close();
            }
        }
    }
    if (!foundAnyLogs) {
        if (nodeId != null) {
            err.println("Can not find information for any containers on " + nodeId);
        } else {
            err.println("Can not find any container information for " + "the application: " + appId);
        }
    }
}