Examples with LogReader org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader used on opensource projects

Example 1 with LogReader

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader in project hadoop by apache.

the class TestAggregatedLogFormat method testForCorruptedAggregatedLogs.

//Test for Corrupted AggregatedLogs. The Logs should not write more data
//if Logvalue.write() is called and the application is still
//appending to logs
@Test
public void testForCorruptedAggregatedLogs() throws Exception {
    Configuration conf = new Configuration();
    File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
    Path remoteAppLogFile = new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
    Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
    ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
    Path t = new Path(srcFileRoot, testContainerId.getApplicationAttemptId().getApplicationId().toString());
    Path srcFilePath = new Path(t, testContainerId.toString());
    long numChars = 950000;
    writeSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot, testContainerId);
    LogReader logReader = new LogReader(conf, remoteAppLogFile);
    LogKey rLogKey = new LogKey();
    DataInputStream dis = logReader.next(rLogKey);
    Writer writer = new StringWriter();
    try {
        LogReader.readAcontainerLogs(dis, writer);
    } catch (Exception e) {
        if (e.toString().contains("NumberFormatException")) {
            Assert.fail("Aggregated logs are corrupted.");
        }
    }
}

Example 2 with LogReader

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader in project hadoop by apache.

the class TestAggregatedLogFormat method testReadAcontainerLog.

private void testReadAcontainerLog(boolean logUploadedTime) throws Exception {
    Configuration conf = new Configuration();
    File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
    Path remoteAppLogFile = new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
    Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
    ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
    Path t = new Path(srcFileRoot, testContainerId.getApplicationAttemptId().getApplicationId().toString());
    Path srcFilePath = new Path(t, testContainerId.toString());
    int numChars = 80000;
    // create a sub-folder under srcFilePath
    // and create file logs in this sub-folder.
    // We only aggregate top level files.
    // So, this log file should be ignored.
    Path subDir = new Path(srcFilePath, "subDir");
    fs.mkdirs(subDir);
    writeSrcFile(subDir, "logs", numChars);
    // create file stderr and stdout in containerLogDir
    writeSrcFile(srcFilePath, "stderr", numChars);
    writeSrcFile(srcFilePath, "stdout", numChars);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);
    LogKey logKey = new LogKey(testContainerId);
    LogValue logValue = new LogValue(Collections.singletonList(srcFileRoot.toString()), testContainerId, ugi.getShortUserName());
    // When we try to open FileInputStream for stderr, it will throw out an IOException.
    // Skip the log aggregation for stderr.
    LogValue spyLogValue = spy(logValue);
    File errorFile = new File((new Path(srcFilePath, "stderr")).toString());
    doThrow(new IOException("Mock can not open FileInputStream")).when(spyLogValue).secureOpenFile(errorFile);
    logWriter.append(logKey, spyLogValue);
    logWriter.close();
    // make sure permission are correct on the file
    FileStatus fsStatus = fs.getFileStatus(remoteAppLogFile);
    Assert.assertEquals("permissions on log aggregation file are wrong", FsPermission.createImmutable((short) 0640), fsStatus.getPermission());
    LogReader logReader = new LogReader(conf, remoteAppLogFile);
    LogKey rLogKey = new LogKey();
    DataInputStream dis = logReader.next(rLogKey);
    Writer writer = new StringWriter();
    if (logUploadedTime) {
        LogReader.readAcontainerLogs(dis, writer, System.currentTimeMillis());
    } else {
        LogReader.readAcontainerLogs(dis, writer);
    }
    // We should only do the log aggregation for stdout.
    // Since we could not open the fileInputStream for stderr, this file is not
    // aggregated.
    String s = writer.toString();
    int expectedLength = "LogType:stdout".length() + (logUploadedTime ? ("\nLog Upload Time:" + Times.format(System.currentTimeMillis())).length() : 0) + ("\nLogLength:" + numChars).length() + "\nLog Contents:\n".length() + numChars + "\n".length() + "\nEnd of LogType:stdout\n".length();
    Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
    Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
    Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
    Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
    Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < numChars; i++) {
        sb.append(filler);
    }
    String expectedContent = sb.toString();
    Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
    Assert.assertEquals(expectedLength, s.length());
}