Examples with PerContainerLogFileInfo org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo used on opensource projects

Example 1 with PerContainerLogFileInfo

use of org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo in project hadoop by apache.

the class LogsCLI method getContainerLogFiles.

private List<Pair<PerContainerLogFileInfo, String>> getContainerLogFiles(Configuration conf, String containerIdStr, String nodeHttpAddress) throws IOException {
    List<Pair<PerContainerLogFileInfo, String>> logFileInfos = new ArrayList<>();
    Client webServiceClient = Client.create();
    try {
        WebResource webResource = webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) + nodeHttpAddress);
        ClientResponse response = webResource.path("ws").path("v1").path("node").path("containers").path(containerIdStr).path("logs").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        if (response.getStatusInfo().getStatusCode() == ClientResponse.Status.OK.getStatusCode()) {
            try {
                JSONArray array = new JSONArray();
                JSONObject json = response.getEntity(JSONObject.class);
                Object logsInfoObj = json.get("containerLogsInfo");
                if (logsInfoObj instanceof JSONObject) {
                    array.put((JSONObject) logsInfoObj);
                } else if (logsInfoObj instanceof JSONArray) {
                    JSONArray logsArray = (JSONArray) logsInfoObj;
                    for (int i = 0; i < logsArray.length(); i++) {
                        array.put(logsArray.getJSONObject(i));
                    }
                }
                for (int i = 0; i < array.length(); i++) {
                    JSONObject log = array.getJSONObject(i);
                    String aggregateType = log.has("logAggregationType") ? log.getString("logAggregationType") : "N/A";
                    Object ob = log.get("containerLogInfo");
                    if (ob instanceof JSONArray) {
                        JSONArray obArray = (JSONArray) ob;
                        for (int j = 0; j < obArray.length(); j++) {
                            logFileInfos.add(new Pair<PerContainerLogFileInfo, String>(generatePerContainerLogFileInfoFromJSON(obArray.getJSONObject(j)), aggregateType));
                        }
                    } else if (ob instanceof JSONObject) {
                        logFileInfos.add(new Pair<PerContainerLogFileInfo, String>(generatePerContainerLogFileInfoFromJSON((JSONObject) ob), aggregateType));
                    }
                }
            } catch (Exception e) {
                System.err.println("Unable to parse json from webservice. Error:");
                System.err.println(e.getMessage());
                throw new IOException(e);
            }
        }
    } catch (ClientHandlerException | UniformInterfaceException ex) {
        System.err.println("Unable to fetch log files list");
        throw new IOException(ex);
    }
    return logFileInfos;
}

Example 2 with PerContainerLogFileInfo

use of org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo in project hadoop by apache.

the class TestNMWebServices method testContainerLogs.

private void testContainerLogs(WebResource r, ContainerId containerId) throws IOException {
    final String containerIdStr = containerId.toString();
    final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
    final ApplicationId appId = appAttemptId.getApplicationId();
    final String appIdStr = appId.toString();
    final String filename = "logfile1";
    final String logMessage = "log message\n";
    nmContext.getApplications().put(appId, new ApplicationImpl(null, "user", appId, null, nmContext));
    MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), new Configuration(), "user", appId, 1);
    container.setState(ContainerState.RUNNING);
    nmContext.getContainers().put(containerId, container);
    // write out log file
    Path path = dirsHandler.getLogPathForWrite(ContainerLaunch.getRelativeContainerLogDir(appIdStr, containerIdStr) + "/" + filename, false);
    File logFile = new File(path.toUri().getPath());
    logFile.deleteOnExit();
    assertTrue("Failed to create log dir", logFile.getParentFile().mkdirs());
    PrintWriter pw = new PrintWriter(logFile);
    pw.print(logMessage);
    pw.close();
    // ask for it
    ClientResponse response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    String responseText = response.getEntity(String.class);
    String responseLogMessage = getLogContext(responseText);
    assertEquals(logMessage, responseLogMessage);
    int fullTextSize = responseLogMessage.getBytes().length;
    // specify how many bytes we should get from logs
    // specify a position number, it would get the first n bytes from
    // container log
    response = r.path(filename).queryParam("size", "5").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(5, responseLogMessage.getBytes().length);
    assertEquals(new String(logMessage.getBytes(), 0, 5), responseLogMessage);
    assertTrue(fullTextSize >= responseLogMessage.getBytes().length);
    // specify the bytes which is larger than the actual file size,
    // we would get the full logs
    response = r.path(filename).queryParam("size", "10000").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(fullTextSize, responseLogMessage.getBytes().length);
    assertEquals(logMessage, responseLogMessage);
    // specify a negative number, it would get the last n bytes from
    // container log
    response = r.path(filename).queryParam("size", "-5").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(5, responseLogMessage.getBytes().length);
    assertEquals(new String(logMessage.getBytes(), logMessage.getBytes().length - 5, 5), responseLogMessage);
    assertTrue(fullTextSize >= responseLogMessage.getBytes().length);
    response = r.path(filename).queryParam("size", "-10000").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals("text/plain; charset=utf-8", response.getType().toString());
    assertEquals(fullTextSize, responseLogMessage.getBytes().length);
    assertEquals(logMessage, responseLogMessage);
    // ask and download it
    response = r.path(filename).queryParam("format", "octet-stream").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(logMessage, responseLogMessage);
    assertEquals(200, response.getStatus());
    assertEquals("application/octet-stream; charset=utf-8", response.getType().toString());
    // specify a invalid format value
    response = r.path(filename).queryParam("format", "123").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    assertEquals("The valid values for the parameter : format are " + WebAppUtils.listSupportedLogContentType(), responseText);
    assertEquals(400, response.getStatus());
    // ask for file that doesn't exist and it will re-direct to
    // the log server
    URI requestURI = r.path("uhhh").getURI();
    String redirectURL = getRedirectURL(requestURI.toString());
    assertTrue(redirectURL != null);
    assertTrue(redirectURL.contains(LOGSERVICEWSADDR));
    // Get container log files' name
    WebResource r1 = resource();
    response = r1.path("ws").path("v1").path("node").path("containers").path(containerIdStr).path("logs").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(200, response.getStatus());
    List<ContainerLogsInfo> responseList = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseList.size() == 1);
    assertEquals(responseList.get(0).getLogType(), ContainerLogAggregationType.LOCAL.toString());
    List<PerContainerLogFileInfo> logMeta = responseList.get(0).getContainerLogsInfo();
    assertTrue(logMeta.size() == 1);
    assertEquals(logMeta.get(0).getFileName(), filename);
    // now create an aggregated log in Remote File system
    File tempLogDir = new File("target", TestNMWebServices.class.getSimpleName() + "temp-log-dir");
    try {
        String aggregatedLogFile = filename + "-aggregated";
        String aggregatedLogMessage = "This is aggregated ;og.";
        TestContainerLogsUtils.createContainerLogFileInRemoteFS(nmContext.getConf(), FileSystem.get(nmContext.getConf()), tempLogDir.getAbsolutePath(), containerId, nmContext.getNodeId(), aggregatedLogFile, "user", aggregatedLogMessage, true);
        r1 = resource();
        response = r1.path("ws").path("v1").path("node").path("containers").path(containerIdStr).path("logs").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(200, response.getStatus());
        responseList = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
        });
        assertEquals(responseList.size(), 2);
        for (ContainerLogsInfo logInfo : responseList) {
            if (logInfo.getLogType().equals(ContainerLogAggregationType.AGGREGATED.toString())) {
                List<PerContainerLogFileInfo> meta = logInfo.getContainerLogsInfo();
                assertTrue(meta.size() == 1);
                assertEquals(meta.get(0).getFileName(), aggregatedLogFile);
            } else {
                assertEquals(logInfo.getLogType(), ContainerLogAggregationType.LOCAL.toString());
                List<PerContainerLogFileInfo> meta = logInfo.getContainerLogsInfo();
                assertTrue(meta.size() == 1);
                assertEquals(meta.get(0).getFileName(), filename);
            }
        }
        // Test whether we could get aggregated log as well
        TestContainerLogsUtils.createContainerLogFileInRemoteFS(nmContext.getConf(), FileSystem.get(nmContext.getConf()), tempLogDir.getAbsolutePath(), containerId, nmContext.getNodeId(), filename, "user", aggregatedLogMessage, true);
        response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
        responseText = response.getEntity(String.class);
        assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.AGGREGATED));
        assertTrue(responseText.contains(aggregatedLogMessage));
        assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.LOCAL));
        assertTrue(responseText.contains(logMessage));
    } finally {
        FileUtil.fullyDelete(tempLogDir);
    }
    // After container is completed, it is removed from nmContext
    nmContext.getContainers().remove(containerId);
    Assert.assertNull(nmContext.getContainers().get(containerId));
    response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    assertTrue(responseText.contains(logMessage));
}

Example 3 with PerContainerLogFileInfo

use of org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo in project hadoop by apache.

the class TestAHSWebServices method testContainerLogsMetaForFinishedApps.

@Test(timeout = 10000)
public void testContainerLogsMetaForFinishedApps() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    String fileName = "syslog";
    String user = "user1";
    String content = "Hello." + containerId1;
    NodeId nodeId = NodeId.newInstance("test host", 100);
    TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, rootLogDir, containerId1, nodeId, fileName, user, content, true);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1.toString()).path("logs").queryParam("user.name", user).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    List<ContainerLogsInfo> responseText = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseText.size() == 1);
    assertEquals(responseText.get(0).getLogType(), ContainerLogAggregationType.AGGREGATED.toString());
    List<PerContainerLogFileInfo> logMeta = responseText.get(0).getContainerLogsInfo();
    assertTrue(logMeta.size() == 1);
    assertEquals(logMeta.get(0).getFileName(), fileName);
    assertEquals(logMeta.get(0).getFileSize(), String.valueOf(content.length()));
}

Example 4 with PerContainerLogFileInfo

use of org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo in project hadoop by apache.

the class TestAHSWebServices method testContainerLogsMetaForRunningApps.

@Test(timeout = 10000)
public void testContainerLogsMetaForRunningApps() throws Exception {
    String user = "user1";
    ApplicationId appId = ApplicationId.newInstance(1234, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    WebResource r = resource();
    // If we specify the NMID, we re-direct the request by using
    // the NM's web address
    URI requestURI = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1.toString()).path("logs").queryParam("user.name", user).queryParam(YarnWebServiceParams.NM_ID, NM_ID).getURI();
    String redirectURL = getRedirectURL(requestURI.toString());
    assertTrue(redirectURL != null);
    assertTrue(redirectURL.contains(NM_WEBADDRESS));
    assertTrue(redirectURL.contains("ws/v1/node/containers"));
    assertTrue(redirectURL.contains(containerId1.toString()));
    assertTrue(redirectURL.contains("/logs"));
    // If we do not specify the NodeId but can get Container information
    // from ATS, we re-direct the request to the node manager
    // who runs the container.
    requestURI = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1.toString()).path("logs").queryParam("user.name", user).getURI();
    redirectURL = getRedirectURL(requestURI.toString());
    assertTrue(redirectURL != null);
    assertTrue(redirectURL.contains("test:1234"));
    assertTrue(redirectURL.contains("ws/v1/node/containers"));
    assertTrue(redirectURL.contains(containerId1.toString()));
    assertTrue(redirectURL.contains("/logs"));
    // If we can not container information from ATS,
    // and not specify nodeId,
    // we would try to get aggregated log meta from remote FileSystem.
    ContainerId containerId1000 = ContainerId.newContainerId(appAttemptId, 1000);
    String fileName = "syslog";
    String content = "Hello." + containerId1000;
    NodeId nodeId = NodeId.newInstance("test host", 100);
    TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, rootLogDir, containerId1000, nodeId, fileName, user, content, true);
    ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1000.toString()).path("logs").queryParam("user.name", user).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    List<ContainerLogsInfo> responseText = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseText.size() == 2);
    for (ContainerLogsInfo logInfo : responseText) {
        if (logInfo.getLogType().equals(ContainerLogAggregationType.AGGREGATED.toString())) {
            List<PerContainerLogFileInfo> logMeta = logInfo.getContainerLogsInfo();
            assertTrue(logMeta.size() == 1);
            assertEquals(logMeta.get(0).getFileName(), fileName);
            assertEquals(logMeta.get(0).getFileSize(), String.valueOf(content.length()));
        } else {
            assertEquals(logInfo.getLogType(), ContainerLogAggregationType.LOCAL.toString());
        }
    }
    // If we can not container information from ATS,
    // and we specify NM id, but can not find NM WebAddress for this nodeId,
    // we would still try to get aggregated log meta from remote FileSystem.
    response = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1000.toString()).path("logs").queryParam(YarnWebServiceParams.NM_ID, "invalid-nm:1234").queryParam("user.name", user).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    responseText = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseText.size() == 2);
    for (ContainerLogsInfo logInfo : responseText) {
        if (logInfo.getLogType().equals(ContainerLogAggregationType.AGGREGATED.toString())) {
            List<PerContainerLogFileInfo> logMeta = logInfo.getContainerLogsInfo();
            assertTrue(logMeta.size() == 1);
            assertEquals(logMeta.get(0).getFileName(), fileName);
            assertEquals(logMeta.get(0).getFileSize(), String.valueOf(content.length()));
        } else {
            assertEquals(logInfo.getLogType(), ContainerLogAggregationType.LOCAL.toString());
        }
    }
}

Example 5 with PerContainerLogFileInfo

use of org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo in project hadoop by apache.

the class NMContainerLogsInfo method getContainerLogsInfo.

private static List<PerContainerLogFileInfo> getContainerLogsInfo(ContainerId id, String remoteUser, Context nmContext) throws YarnException {
    List<PerContainerLogFileInfo> logFiles = new ArrayList<>();
    List<File> logDirs = ContainerLogsUtils.getContainerLogDirs(id, remoteUser, nmContext);
    for (File containerLogsDir : logDirs) {
        File[] logs = containerLogsDir.listFiles();
        if (logs != null) {
            for (File log : logs) {
                if (log.isFile()) {
                    PerContainerLogFileInfo logMeta = new PerContainerLogFileInfo(log.getName(), Long.toString(log.length()), Times.format(log.lastModified()));
                    logFiles.add(logMeta);
                }
            }
        }
    }
    return logFiles;
}