Examples with LogKey org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey used on opensource projects

Example 6 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class TestAggregatedLogFormat method testForCorruptedAggregatedLogs.

//Test for Corrupted AggregatedLogs. The Logs should not write more data
//if Logvalue.write() is called and the application is still
//appending to logs
@Test
public void testForCorruptedAggregatedLogs() throws Exception {
    Configuration conf = new Configuration();
    File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
    Path remoteAppLogFile = new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
    Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
    ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
    Path t = new Path(srcFileRoot, testContainerId.getApplicationAttemptId().getApplicationId().toString());
    Path srcFilePath = new Path(t, testContainerId.toString());
    long numChars = 950000;
    writeSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot, testContainerId);
    LogReader logReader = new LogReader(conf, remoteAppLogFile);
    LogKey rLogKey = new LogKey();
    DataInputStream dis = logReader.next(rLogKey);
    Writer writer = new StringWriter();
    try {
        LogReader.readAcontainerLogs(dis, writer);
    } catch (Exception e) {
        if (e.toString().contains("NumberFormatException")) {
            Assert.fail("Aggregated logs are corrupted.");
        }
    }
}

Example 7 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class TestAggregatedLogFormat method testReadAcontainerLog.

private void testReadAcontainerLog(boolean logUploadedTime) throws Exception {
    Configuration conf = new Configuration();
    File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
    Path remoteAppLogFile = new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
    Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
    ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
    Path t = new Path(srcFileRoot, testContainerId.getApplicationAttemptId().getApplicationId().toString());
    Path srcFilePath = new Path(t, testContainerId.toString());
    int numChars = 80000;
    // create a sub-folder under srcFilePath
    // and create file logs in this sub-folder.
    // We only aggregate top level files.
    // So, this log file should be ignored.
    Path subDir = new Path(srcFilePath, "subDir");
    fs.mkdirs(subDir);
    writeSrcFile(subDir, "logs", numChars);
    // create file stderr and stdout in containerLogDir
    writeSrcFile(srcFilePath, "stderr", numChars);
    writeSrcFile(srcFilePath, "stdout", numChars);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);
    LogKey logKey = new LogKey(testContainerId);
    LogValue logValue = new LogValue(Collections.singletonList(srcFileRoot.toString()), testContainerId, ugi.getShortUserName());
    // When we try to open FileInputStream for stderr, it will throw out an IOException.
    // Skip the log aggregation for stderr.
    LogValue spyLogValue = spy(logValue);
    File errorFile = new File((new Path(srcFilePath, "stderr")).toString());
    doThrow(new IOException("Mock can not open FileInputStream")).when(spyLogValue).secureOpenFile(errorFile);
    logWriter.append(logKey, spyLogValue);
    logWriter.close();
    // make sure permission are correct on the file
    FileStatus fsStatus = fs.getFileStatus(remoteAppLogFile);
    Assert.assertEquals("permissions on log aggregation file are wrong", FsPermission.createImmutable((short) 0640), fsStatus.getPermission());
    LogReader logReader = new LogReader(conf, remoteAppLogFile);
    LogKey rLogKey = new LogKey();
    DataInputStream dis = logReader.next(rLogKey);
    Writer writer = new StringWriter();
    if (logUploadedTime) {
        LogReader.readAcontainerLogs(dis, writer, System.currentTimeMillis());
    } else {
        LogReader.readAcontainerLogs(dis, writer);
    }
    // We should only do the log aggregation for stdout.
    // Since we could not open the fileInputStream for stderr, this file is not
    // aggregated.
    String s = writer.toString();
    int expectedLength = "LogType:stdout".length() + (logUploadedTime ? ("\nLog Upload Time:" + Times.format(System.currentTimeMillis())).length() : 0) + ("\nLogLength:" + numChars).length() + "\nLog Contents:\n".length() + numChars + "\n".length() + "\nEnd of LogType:stdout\n".length();
    Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
    Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
    Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
    Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
    Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < numChars; i++) {
        sb.append(filler);
    }
    String expectedContent = sb.toString();
    Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
    Assert.assertEquals(expectedLength, s.length());
}

Example 8 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class TestLogAggregationService method verifyContainerLogs.

// expectedContainerIds is the minimal set of containers to check.
// The actual list of containers could be more than that.
// Verify the size of the actual list is in the range of
// [minNumOfContainers, maxNumOfContainers].
private LogFileStatusInLastCycle verifyContainerLogs(LogAggregationService logAggregationService, ApplicationId appId, ContainerId[] expectedContainerIds, int minNumOfContainers, int maxNumOfContainers, String[] logFiles, int numOfLogsPerContainer, boolean multiLogs) throws IOException {
    Path appLogDir = logAggregationService.getRemoteAppLogDir(appId, this.user);
    RemoteIterator<FileStatus> nodeFiles = null;
    try {
        Path qualifiedLogDir = FileContext.getFileContext(this.conf).makeQualified(appLogDir);
        nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), this.conf).listStatus(appLogDir);
    } catch (FileNotFoundException fnf) {
        Assert.fail("Should have log files");
    }
    if (numOfLogsPerContainer == 0) {
        Assert.assertTrue(!nodeFiles.hasNext());
        return null;
    }
    Assert.assertTrue(nodeFiles.hasNext());
    FileStatus targetNodeFile = null;
    if (!multiLogs) {
        targetNodeFile = nodeFiles.next();
        Assert.assertTrue(targetNodeFile.getPath().getName().equals(LogAggregationUtils.getNodeString(logAggregationService.getNodeId())));
    } else {
        long fileCreateTime = 0;
        while (nodeFiles.hasNext()) {
            FileStatus nodeFile = nodeFiles.next();
            if (!nodeFile.getPath().getName().contains(LogAggregationUtils.TMP_FILE_SUFFIX)) {
                long time = Long.parseLong(nodeFile.getPath().getName().split("_")[2]);
                if (time > fileCreateTime) {
                    targetNodeFile = nodeFile;
                    fileCreateTime = time;
                }
            }
        }
        String[] fileName = targetNodeFile.getPath().getName().split("_");
        Assert.assertTrue(fileName.length == 3);
        Assert.assertEquals(fileName[0] + ":" + fileName[1], logAggregationService.getNodeId().toString());
    }
    AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(this.conf, targetNodeFile.getPath());
    Assert.assertEquals(this.user, reader.getApplicationOwner());
    verifyAcls(reader.getApplicationAcls());
    List<String> fileTypes = new ArrayList<String>();
    try {
        Map<String, Map<String, String>> logMap = new HashMap<String, Map<String, String>>();
        DataInputStream valueStream;
        LogKey key = new LogKey();
        valueStream = reader.next(key);
        while (valueStream != null) {
            LOG.info("Found container " + key.toString());
            Map<String, String> perContainerMap = new HashMap<String, String>();
            logMap.put(key.toString(), perContainerMap);
            while (true) {
                try {
                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    PrintStream ps = new PrintStream(baos);
                    LogReader.readAContainerLogsForALogType(valueStream, ps);
                    String[] writtenLines = baos.toString().split(System.getProperty("line.separator"));
                    Assert.assertEquals("LogType:", writtenLines[0].substring(0, 8));
                    String fileType = writtenLines[0].substring(8);
                    fileTypes.add(fileType);
                    Assert.assertEquals("LogLength:", writtenLines[1].substring(0, 10));
                    String fileLengthStr = writtenLines[1].substring(10);
                    long fileLength = Long.parseLong(fileLengthStr);
                    Assert.assertEquals("Log Contents:", writtenLines[2].substring(0, 13));
                    String logContents = StringUtils.join(Arrays.copyOfRange(writtenLines, 3, writtenLines.length), "\n");
                    perContainerMap.put(fileType, logContents);
                    LOG.info("LogType:" + fileType);
                    LOG.info("LogLength:" + fileLength);
                    LOG.info("Log Contents:\n" + perContainerMap.get(fileType));
                } catch (EOFException eof) {
                    break;
                }
            }
            // Next container
            key = new LogKey();
            valueStream = reader.next(key);
        }
        // 1 for each container
        Assert.assertTrue("number of containers with logs should be at least " + minNumOfContainers, logMap.size() >= minNumOfContainers);
        Assert.assertTrue("number of containers with logs should be at most " + minNumOfContainers, logMap.size() <= maxNumOfContainers);
        for (ContainerId cId : expectedContainerIds) {
            String containerStr = cId.toString();
            Map<String, String> thisContainerMap = logMap.remove(containerStr);
            Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size());
            for (String fileType : logFiles) {
                String expectedValue = containerStr + " Hello " + fileType + "!\nEnd of LogType:" + fileType;
                LOG.info("Expected log-content : " + new String(expectedValue));
                String foundValue = thisContainerMap.remove(fileType);
                Assert.assertNotNull(cId + " " + fileType + " not present in aggregated log-file!", foundValue);
                Assert.assertEquals(expectedValue, foundValue);
            }
            Assert.assertEquals(0, thisContainerMap.size());
        }
        Assert.assertTrue("number of remaining containers should be at least " + (minNumOfContainers - expectedContainerIds.length), logMap.size() >= minNumOfContainers - expectedContainerIds.length);
        Assert.assertTrue("number of remaining containers should be at most " + (maxNumOfContainers - expectedContainerIds.length), logMap.size() <= maxNumOfContainers - expectedContainerIds.length);
        return new LogFileStatusInLastCycle(targetNodeFile.getPath().getName(), fileTypes);
    } finally {
        reader.close();
    }
}

Example 9 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class LogCLIHelpers method listContainerLogs.

@Private
public Set<String> listContainerLogs(ContainerLogsRequest options) throws IOException {
    Set<String> logTypes = new HashSet<String>();
    ApplicationId appId = options.getAppId();
    String appOwner = options.getAppOwner();
    String nodeId = options.getNodeId();
    String containerIdStr = options.getContainerId();
    boolean getAllContainers = (containerIdStr == null);
    String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId);
    RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(appId, appOwner);
    if (nodeFiles == null) {
        return logTypes;
    }
    while (nodeFiles.hasNext()) {
        FileStatus thisNodeFile = nodeFiles.next();
        if (nodeIdStr != null) {
            if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
                continue;
            }
        }
        if (!thisNodeFile.getPath().getName().endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
            AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath());
            try {
                DataInputStream valueStream;
                LogKey key = new LogKey();
                valueStream = reader.next(key);
                while (valueStream != null) {
                    if (getAllContainers || (key.toString().equals(containerIdStr))) {
                        while (true) {
                            try {
                                String logFile = LogReader.readContainerMetaDataAndSkipData(valueStream).getFirst();
                                logTypes.add(logFile);
                            } catch (EOFException eof) {
                                break;
                            }
                        }
                        if (!getAllContainers) {
                            break;
                        }
                    }
                    // Next container
                    key = new LogKey();
                    valueStream = reader.next(key);
                }
            } finally {
                reader.close();
            }
        }
    }
    return logTypes;
}

Example 10 with LogKey

use of org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey in project hadoop by apache.

the class LogToolUtils method getContainerLogMetaFromRemoteFS.

/**
   * Return a list of {@link ContainerLogMeta} for a container
   * from Remote FileSystem.
   *
   * @param conf the configuration
   * @param appId the applicationId
   * @param containerIdStr the containerId
   * @param nodeId the nodeId
   * @param appOwner the application owner
   * @return a list of {@link ContainerLogMeta}
   * @throws IOException if there is no available log file
   */
public static List<ContainerLogMeta> getContainerLogMetaFromRemoteFS(Configuration conf, ApplicationId appId, String containerIdStr, String nodeId, String appOwner) throws IOException {
    List<ContainerLogMeta> containersLogMeta = new ArrayList<>();
    boolean getAllContainers = (containerIdStr == null);
    String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId);
    RemoteIterator<FileStatus> nodeFiles = LogAggregationUtils.getRemoteNodeFileDir(conf, appId, appOwner);
    if (nodeFiles == null) {
        throw new IOException("There is no available log fils for " + "application:" + appId);
    }
    while (nodeFiles.hasNext()) {
        FileStatus thisNodeFile = nodeFiles.next();
        if (nodeIdStr != null) {
            if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
                continue;
            }
        }
        if (!thisNodeFile.getPath().getName().endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
            AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(conf, thisNodeFile.getPath());
            try {
                DataInputStream valueStream;
                LogKey key = new LogKey();
                valueStream = reader.next(key);
                while (valueStream != null) {
                    if (getAllContainers || (key.toString().equals(containerIdStr))) {
                        ContainerLogMeta containerLogMeta = new ContainerLogMeta(key.toString(), thisNodeFile.getPath().getName());
                        while (true) {
                            try {
                                Pair<String, String> logMeta = LogReader.readContainerMetaDataAndSkipData(valueStream);
                                containerLogMeta.addLogMeta(logMeta.getFirst(), logMeta.getSecond(), Times.format(thisNodeFile.getModificationTime()));
                            } catch (EOFException eof) {
                                break;
                            }
                        }
                        containersLogMeta.add(containerLogMeta);
                        if (!getAllContainers) {
                            break;
                        }
                    }
                    // Next container
                    key = new LogKey();
                    valueStream = reader.next(key);
                }
            } finally {
                reader.close();
            }
        }
    }
    return containersLogMeta;
}