Examples with ContainerLogsInfo org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo used on opensource projects

Example 1 with ContainerLogsInfo

use of org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo in project hadoop by apache.

the class TestNMWebServices method testContainerLogs.

private void testContainerLogs(WebResource r, ContainerId containerId) throws IOException {
    final String containerIdStr = containerId.toString();
    final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
    final ApplicationId appId = appAttemptId.getApplicationId();
    final String appIdStr = appId.toString();
    final String filename = "logfile1";
    final String logMessage = "log message\n";
    nmContext.getApplications().put(appId, new ApplicationImpl(null, "user", appId, null, nmContext));
    MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), new Configuration(), "user", appId, 1);
    container.setState(ContainerState.RUNNING);
    nmContext.getContainers().put(containerId, container);
    // write out log file
    Path path = dirsHandler.getLogPathForWrite(ContainerLaunch.getRelativeContainerLogDir(appIdStr, containerIdStr) + "/" + filename, false);
    File logFile = new File(path.toUri().getPath());
    logFile.deleteOnExit();
    assertTrue("Failed to create log dir", logFile.getParentFile().mkdirs());
    PrintWriter pw = new PrintWriter(logFile);
    pw.print(logMessage);
    pw.close();
    // ask for it
    ClientResponse response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    String responseText = response.getEntity(String.class);
    String responseLogMessage = getLogContext(responseText);
    assertEquals(logMessage, responseLogMessage);
    int fullTextSize = responseLogMessage.getBytes().length;
    // specify how many bytes we should get from logs
    // specify a position number, it would get the first n bytes from
    // container log
    response = r.path(filename).queryParam("size", "5").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(5, responseLogMessage.getBytes().length);
    assertEquals(new String(logMessage.getBytes(), 0, 5), responseLogMessage);
    assertTrue(fullTextSize >= responseLogMessage.getBytes().length);
    // specify the bytes which is larger than the actual file size,
    // we would get the full logs
    response = r.path(filename).queryParam("size", "10000").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(fullTextSize, responseLogMessage.getBytes().length);
    assertEquals(logMessage, responseLogMessage);
    // specify a negative number, it would get the last n bytes from
    // container log
    response = r.path(filename).queryParam("size", "-5").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(5, responseLogMessage.getBytes().length);
    assertEquals(new String(logMessage.getBytes(), logMessage.getBytes().length - 5, 5), responseLogMessage);
    assertTrue(fullTextSize >= responseLogMessage.getBytes().length);
    response = r.path(filename).queryParam("size", "-10000").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals("text/plain; charset=utf-8", response.getType().toString());
    assertEquals(fullTextSize, responseLogMessage.getBytes().length);
    assertEquals(logMessage, responseLogMessage);
    // ask and download it
    response = r.path(filename).queryParam("format", "octet-stream").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    responseLogMessage = getLogContext(responseText);
    assertEquals(logMessage, responseLogMessage);
    assertEquals(200, response.getStatus());
    assertEquals("application/octet-stream; charset=utf-8", response.getType().toString());
    // specify a invalid format value
    response = r.path(filename).queryParam("format", "123").accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    assertEquals("The valid values for the parameter : format are " + WebAppUtils.listSupportedLogContentType(), responseText);
    assertEquals(400, response.getStatus());
    // ask for file that doesn't exist and it will re-direct to
    // the log server
    URI requestURI = r.path("uhhh").getURI();
    String redirectURL = getRedirectURL(requestURI.toString());
    assertTrue(redirectURL != null);
    assertTrue(redirectURL.contains(LOGSERVICEWSADDR));
    // Get container log files' name
    WebResource r1 = resource();
    response = r1.path("ws").path("v1").path("node").path("containers").path(containerIdStr).path("logs").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(200, response.getStatus());
    List<ContainerLogsInfo> responseList = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseList.size() == 1);
    assertEquals(responseList.get(0).getLogType(), ContainerLogAggregationType.LOCAL.toString());
    List<PerContainerLogFileInfo> logMeta = responseList.get(0).getContainerLogsInfo();
    assertTrue(logMeta.size() == 1);
    assertEquals(logMeta.get(0).getFileName(), filename);
    // now create an aggregated log in Remote File system
    File tempLogDir = new File("target", TestNMWebServices.class.getSimpleName() + "temp-log-dir");
    try {
        String aggregatedLogFile = filename + "-aggregated";
        String aggregatedLogMessage = "This is aggregated ;og.";
        TestContainerLogsUtils.createContainerLogFileInRemoteFS(nmContext.getConf(), FileSystem.get(nmContext.getConf()), tempLogDir.getAbsolutePath(), containerId, nmContext.getNodeId(), aggregatedLogFile, "user", aggregatedLogMessage, true);
        r1 = resource();
        response = r1.path("ws").path("v1").path("node").path("containers").path(containerIdStr).path("logs").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(200, response.getStatus());
        responseList = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
        });
        assertEquals(responseList.size(), 2);
        for (ContainerLogsInfo logInfo : responseList) {
            if (logInfo.getLogType().equals(ContainerLogAggregationType.AGGREGATED.toString())) {
                List<PerContainerLogFileInfo> meta = logInfo.getContainerLogsInfo();
                assertTrue(meta.size() == 1);
                assertEquals(meta.get(0).getFileName(), aggregatedLogFile);
            } else {
                assertEquals(logInfo.getLogType(), ContainerLogAggregationType.LOCAL.toString());
                List<PerContainerLogFileInfo> meta = logInfo.getContainerLogsInfo();
                assertTrue(meta.size() == 1);
                assertEquals(meta.get(0).getFileName(), filename);
            }
        }
        // Test whether we could get aggregated log as well
        TestContainerLogsUtils.createContainerLogFileInRemoteFS(nmContext.getConf(), FileSystem.get(nmContext.getConf()), tempLogDir.getAbsolutePath(), containerId, nmContext.getNodeId(), filename, "user", aggregatedLogMessage, true);
        response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
        responseText = response.getEntity(String.class);
        assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.AGGREGATED));
        assertTrue(responseText.contains(aggregatedLogMessage));
        assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.LOCAL));
        assertTrue(responseText.contains(logMessage));
    } finally {
        FileUtil.fullyDelete(tempLogDir);
    }
    // After container is completed, it is removed from nmContext
    nmContext.getContainers().remove(containerId);
    Assert.assertNull(nmContext.getContainers().get(containerId));
    response = r.path(filename).accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
    responseText = response.getEntity(String.class);
    assertTrue(responseText.contains(logMessage));
}

Example 2 with ContainerLogsInfo

use of org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo in project hadoop by apache.

the class NMWebServices method getContainerLogsInfo.

/**
   * Returns log file's name as well as current file size for a container.
   *
   * @param hsr
   *    HttpServletRequest
   * @param res
   *    HttpServletResponse
   * @param containerIdStr
   *    The container ID
   * @return
   *    The log file's name and current file size
   */
@GET
@Path("/containers/{containerid}/logs")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public Response getContainerLogsInfo(@javax.ws.rs.core.Context HttpServletRequest hsr, @javax.ws.rs.core.Context HttpServletResponse res, @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr) {
    ContainerId containerId = null;
    init();
    try {
        containerId = ContainerId.fromString(containerIdStr);
    } catch (IllegalArgumentException ex) {
        throw new BadRequestException("invalid container id, " + containerIdStr);
    }
    try {
        List<ContainerLogsInfo> containersLogsInfo = new ArrayList<>();
        containersLogsInfo.add(new NMContainerLogsInfo(this.nmContext, containerId, hsr.getRemoteUser(), ContainerLogAggregationType.LOCAL));
        // check whether we have aggregated logs in RemoteFS. If exists, show the
        // the log meta for the aggregated logs as well.
        ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
        Application app = this.nmContext.getApplications().get(appId);
        String appOwner = app == null ? null : app.getUser();
        try {
            List<ContainerLogMeta> containerLogMeta = LogToolUtils.getContainerLogMetaFromRemoteFS(this.nmContext.getConf(), appId, containerIdStr, this.nmContext.getNodeId().toString(), appOwner);
            if (!containerLogMeta.isEmpty()) {
                for (ContainerLogMeta logMeta : containerLogMeta) {
                    containersLogsInfo.add(new ContainerLogsInfo(logMeta, ContainerLogAggregationType.AGGREGATED));
                }
            }
        } catch (IOException ex) {
            // Skip it and do nothing
            if (LOG.isDebugEnabled()) {
                LOG.debug(ex.getMessage());
            }
        }
        GenericEntity<List<ContainerLogsInfo>> meta = new GenericEntity<List<ContainerLogsInfo>>(containersLogsInfo) {
        };
        ResponseBuilder resp = Response.ok(meta);
        // Sending the X-Content-Type-Options response header with the value
        // nosniff will prevent Internet Explorer from MIME-sniffing a response
        // away from the declared content-type.
        resp.header("X-Content-Type-Options", "nosniff");
        return resp.build();
    } catch (Exception ex) {
        if (redirectWSUrl == null || redirectWSUrl.isEmpty()) {
            throw new WebApplicationException(ex);
        }
        // redirect the request to the configured log server
        String redirectURI = "/containers/" + containerIdStr + "/logs";
        return createRedirectResponse(hsr, redirectWSUrl, redirectURI);
    }
}

Example 3 with ContainerLogsInfo

use of org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo in project hadoop by apache.

the class AHSWebServices method getContainerLogMeta.

private Response getContainerLogMeta(ApplicationId appId, String appOwner, final String nodeId, final String containerIdStr, boolean emptyLocalContainerLogMeta) {
    try {
        List<ContainerLogMeta> containerLogMeta = LogToolUtils.getContainerLogMetaFromRemoteFS(conf, appId, containerIdStr, nodeId, appOwner);
        if (containerLogMeta.isEmpty()) {
            throw new NotFoundException("Can not get log meta for container: " + containerIdStr);
        }
        List<ContainerLogsInfo> containersLogsInfo = new ArrayList<>();
        for (ContainerLogMeta meta : containerLogMeta) {
            ContainerLogsInfo logInfo = new ContainerLogsInfo(meta, ContainerLogAggregationType.AGGREGATED);
            containersLogsInfo.add(logInfo);
        }
        if (emptyLocalContainerLogMeta) {
            ContainerLogMeta emptyMeta = new ContainerLogMeta(containerIdStr, "N/A");
            ContainerLogsInfo empty = new ContainerLogsInfo(emptyMeta, ContainerLogAggregationType.LOCAL);
            containersLogsInfo.add(empty);
        }
        GenericEntity<List<ContainerLogsInfo>> meta = new GenericEntity<List<ContainerLogsInfo>>(containersLogsInfo) {
        };
        ResponseBuilder response = Response.ok(meta);
        // Sending the X-Content-Type-Options response header with the value
        // nosniff will prevent Internet Explorer from MIME-sniffing a response
        // away from the declared content-type.
        response.header("X-Content-Type-Options", "nosniff");
        return response.build();
    } catch (Exception ex) {
        throw new WebApplicationException(ex);
    }
}

Example 4 with ContainerLogsInfo

use of org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo in project hadoop by apache.

the class TestAHSWebServices method testContainerLogsMetaForFinishedApps.

@Test(timeout = 10000)
public void testContainerLogsMetaForFinishedApps() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    String fileName = "syslog";
    String user = "user1";
    String content = "Hello." + containerId1;
    NodeId nodeId = NodeId.newInstance("test host", 100);
    TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, rootLogDir, containerId1, nodeId, fileName, user, content, true);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1.toString()).path("logs").queryParam("user.name", user).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    List<ContainerLogsInfo> responseText = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseText.size() == 1);
    assertEquals(responseText.get(0).getLogType(), ContainerLogAggregationType.AGGREGATED.toString());
    List<PerContainerLogFileInfo> logMeta = responseText.get(0).getContainerLogsInfo();
    assertTrue(logMeta.size() == 1);
    assertEquals(logMeta.get(0).getFileName(), fileName);
    assertEquals(logMeta.get(0).getFileSize(), String.valueOf(content.length()));
}

Example 5 with ContainerLogsInfo

use of org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo in project hadoop by apache.

the class TestAHSWebServices method testContainerLogsMetaForRunningApps.

@Test(timeout = 10000)
public void testContainerLogsMetaForRunningApps() throws Exception {
    String user = "user1";
    ApplicationId appId = ApplicationId.newInstance(1234, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
    WebResource r = resource();
    // If we specify the NMID, we re-direct the request by using
    // the NM's web address
    URI requestURI = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1.toString()).path("logs").queryParam("user.name", user).queryParam(YarnWebServiceParams.NM_ID, NM_ID).getURI();
    String redirectURL = getRedirectURL(requestURI.toString());
    assertTrue(redirectURL != null);
    assertTrue(redirectURL.contains(NM_WEBADDRESS));
    assertTrue(redirectURL.contains("ws/v1/node/containers"));
    assertTrue(redirectURL.contains(containerId1.toString()));
    assertTrue(redirectURL.contains("/logs"));
    // If we do not specify the NodeId but can get Container information
    // from ATS, we re-direct the request to the node manager
    // who runs the container.
    requestURI = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1.toString()).path("logs").queryParam("user.name", user).getURI();
    redirectURL = getRedirectURL(requestURI.toString());
    assertTrue(redirectURL != null);
    assertTrue(redirectURL.contains("test:1234"));
    assertTrue(redirectURL.contains("ws/v1/node/containers"));
    assertTrue(redirectURL.contains(containerId1.toString()));
    assertTrue(redirectURL.contains("/logs"));
    // If we can not container information from ATS,
    // and not specify nodeId,
    // we would try to get aggregated log meta from remote FileSystem.
    ContainerId containerId1000 = ContainerId.newContainerId(appAttemptId, 1000);
    String fileName = "syslog";
    String content = "Hello." + containerId1000;
    NodeId nodeId = NodeId.newInstance("test host", 100);
    TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, rootLogDir, containerId1000, nodeId, fileName, user, content, true);
    ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1000.toString()).path("logs").queryParam("user.name", user).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    List<ContainerLogsInfo> responseText = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseText.size() == 2);
    for (ContainerLogsInfo logInfo : responseText) {
        if (logInfo.getLogType().equals(ContainerLogAggregationType.AGGREGATED.toString())) {
            List<PerContainerLogFileInfo> logMeta = logInfo.getContainerLogsInfo();
            assertTrue(logMeta.size() == 1);
            assertEquals(logMeta.get(0).getFileName(), fileName);
            assertEquals(logMeta.get(0).getFileSize(), String.valueOf(content.length()));
        } else {
            assertEquals(logInfo.getLogType(), ContainerLogAggregationType.LOCAL.toString());
        }
    }
    // If we can not container information from ATS,
    // and we specify NM id, but can not find NM WebAddress for this nodeId,
    // we would still try to get aggregated log meta from remote FileSystem.
    response = r.path("ws").path("v1").path("applicationhistory").path("containers").path(containerId1000.toString()).path("logs").queryParam(YarnWebServiceParams.NM_ID, "invalid-nm:1234").queryParam("user.name", user).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    responseText = response.getEntity(new GenericType<List<ContainerLogsInfo>>() {
    });
    assertTrue(responseText.size() == 2);
    for (ContainerLogsInfo logInfo : responseText) {
        if (logInfo.getLogType().equals(ContainerLogAggregationType.AGGREGATED.toString())) {
            List<PerContainerLogFileInfo> logMeta = logInfo.getContainerLogsInfo();
            assertTrue(logMeta.size() == 1);
            assertEquals(logMeta.get(0).getFileName(), fileName);
            assertEquals(logMeta.get(0).getFileSize(), String.valueOf(content.length()));
        } else {
            assertEquals(logInfo.getLogType(), ContainerLogAggregationType.LOCAL.toString());
        }
    }
}