Examples with WorkerLogs org.apache.storm.daemon.logviewer.utils.WorkerLogs used on opensource projects

Example 1 with WorkerLogs

use of org.apache.storm.daemon.logviewer.utils.WorkerLogs in project storm by apache.

the class LogviewerServer method main.

/**
 * Main method to start the server.
 */
public static void main(String[] args) throws Exception {
    Utils.setupDefaultUncaughtExceptionHandler();
    Map<String, Object> conf = ConfigUtils.readStormConfig();
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    String logRoot = ConfigUtils.workerArtifactsRoot(conf);
    File logRootDir = new File(logRoot);
    logRootDir.mkdirs();
    WorkerLogs workerLogs = new WorkerLogs(conf, logRootDir.toPath(), metricsRegistry);
    DirectoryCleaner directoryCleaner = new DirectoryCleaner(metricsRegistry);
    try (LogviewerServer server = new LogviewerServer(conf, metricsRegistry);
        LogCleaner logCleaner = new LogCleaner(conf, workerLogs, directoryCleaner, logRootDir.toPath(), metricsRegistry)) {
        metricsRegistry.startMetricsReporters(conf);
        Utils.addShutdownHookWithForceKillIn1Sec(() -> {
            server.meterShutdownCalls.mark();
            metricsRegistry.stopMetricsReporters();
            server.close();
        });
        logCleaner.start();
        server.start();
        server.awaitTermination();
    }
}

Example 2 with WorkerLogs

use of org.apache.storm.daemon.logviewer.utils.WorkerLogs in project storm by apache.

the class LogviewerLogDownloadHandlerTest method createHandlerTraversalTests.

private LogviewerLogDownloadHandler createHandlerTraversalTests(Path rootPath) throws IOException {
    Path daemonLogRoot = rootPath.resolve("logs");
    Path fileOutsideDaemonRoot = rootPath.resolve("evil.sh");
    Path workerLogRoot = daemonLogRoot.resolve("workers-artifacts");
    Path daemonFile = daemonLogRoot.resolve("nimbus.log");
    Path topoA = workerLogRoot.resolve("topoA");
    Path file1 = topoA.resolve("1111").resolve("worker.log");
    Path file2 = topoA.resolve("2222").resolve("worker.log");
    Path file3 = workerLogRoot.resolve("topoB").resolve("1111").resolve("worker.log");
    Files.createDirectories(file1.getParent());
    Files.createDirectories(file2.getParent());
    Files.createDirectories(file3.getParent());
    Files.createFile(file1);
    Files.createFile(file2);
    Files.createFile(file3);
    Files.createFile(fileOutsideDaemonRoot);
    Files.createFile(daemonFile);
    Map<String, Object> stormConf = Utils.readStormConfig();
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    return new LogviewerLogDownloadHandler(workerLogRoot.toString(), daemonLogRoot.toString(), new WorkerLogs(stormConf, workerLogRoot, metricsRegistry), new ResourceAuthorizer(stormConf), metricsRegistry);
}

Example 3 with WorkerLogs

use of org.apache.storm.daemon.logviewer.utils.WorkerLogs in project storm by apache.

the class LogviewerLogPageHandlerTest method testListLogFiles.

/**
 * list-log-files filter selects the correct log files to return.
 */
@Test
public void testListLogFiles() throws IOException {
    String rootPath = Files.createTempDirectory("workers-artifacts").toFile().getCanonicalPath();
    File file1 = new File(String.join(File.separator, rootPath, "topoA", "1111"), "worker.log");
    File file2 = new File(String.join(File.separator, rootPath, "topoA", "2222"), "worker.log");
    File file3 = new File(String.join(File.separator, rootPath, "topoB", "1111"), "worker.log");
    file1.getParentFile().mkdirs();
    file2.getParentFile().mkdirs();
    file3.getParentFile().mkdirs();
    file1.createNewFile();
    file2.createNewFile();
    file3.createNewFile();
    String origin = "www.origin.server.net";
    Map<String, Object> stormConf = Utils.readStormConfig();
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    LogviewerLogPageHandler handler = new LogviewerLogPageHandler(rootPath, rootPath, new WorkerLogs(stormConf, Paths.get(rootPath), metricsRegistry), new ResourceAuthorizer(stormConf), metricsRegistry);
    final Response expectedAll = LogviewerResponseBuilder.buildSuccessJsonResponse(Lists.newArrayList("topoA/port1/worker.log", "topoA/port2/worker.log", "topoB/port1/worker.log"), null, origin);
    final Response expectedFilterPort = LogviewerResponseBuilder.buildSuccessJsonResponse(Lists.newArrayList("topoA/port1/worker.log", "topoB/port1/worker.log"), null, origin);
    final Response expectedFilterTopoId = LogviewerResponseBuilder.buildSuccessJsonResponse(Lists.newArrayList("topoB/port1/worker.log"), null, origin);
    final Response returnedAll = handler.listLogFiles("user", null, null, null, origin);
    final Response returnedFilterPort = handler.listLogFiles("user", 1111, null, null, origin);
    final Response returnedFilterTopoId = handler.listLogFiles("user", null, "topoB", null, origin);
    Utils.forceDelete(rootPath);
    assertEqualsJsonResponse(expectedAll, returnedAll, List.class);
    assertEqualsJsonResponse(expectedFilterPort, returnedFilterPort, List.class);
    assertEqualsJsonResponse(expectedFilterTopoId, returnedFilterTopoId, List.class);
}

Example 4 with WorkerLogs

use of org.apache.storm.daemon.logviewer.utils.WorkerLogs in project storm by apache.

the class LogviewerLogPageHandler method logPage.

/**
 * Provides a worker log file to view, starting from the specified position
 * or default starting position of the most recent page.
 *
 * @param fileName file to view
 * @param start start offset, or null if the most recent page is desired
 * @param length length to read in this page, or null if default page length is desired
 * @param grep search string if request is a result of the search, can be null
 * @param user username
 * @return HTML view page of worker log
 */
public Response logPage(String fileName, Integer start, Integer length, String grep, String user) throws IOException, InvalidRequestException {
    Path rawFile = logRoot.resolve(fileName);
    Path absFile = rawFile.toAbsolutePath().normalize();
    if (!absFile.startsWith(logRoot) || !rawFile.normalize().toString().equals(rawFile.toString())) {
        // Ensure filename doesn't contain ../ parts
        return LogviewerResponseBuilder.buildResponsePageNotFound();
    }
    if (resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) {
        workerLogs.setLogFilePermission(fileName);
        Path topoDir = absFile.getParent().getParent();
        if (absFile.toFile().exists()) {
            SortedSet<Path> logFiles;
            try {
                logFiles = Arrays.stream(topoDir.toFile().listFiles()).flatMap(Unchecked.function(portDir -> directoryCleaner.getFilesForDir(portDir.toPath()).stream())).filter(Files::isRegularFile).collect(toCollection(TreeSet::new));
            } catch (UncheckedIOException e) {
                throw e.getCause();
            }
            List<String> reorderedFilesStr = logFiles.stream().map(WorkerLogs::getTopologyPortWorkerLog).filter(fileStr -> !StringUtils.equals(fileName, fileStr)).collect(toList());
            reorderedFilesStr.add(fileName);
            length = length != null ? Math.min(10485760, length) : LogviewerConstant.DEFAULT_BYTES_PER_PAGE;
            final boolean isZipFile = absFile.getFileName().toString().endsWith(".gz");
            long fileLength = getFileLength(absFile.toFile(), isZipFile);
            if (start == null) {
                start = Long.valueOf(fileLength - length).intValue();
            }
            String logString = isTxtFile(fileName) ? escapeHtml(pageFile(absFile.toString(), isZipFile, fileLength, start, length)) : escapeHtml("This is a binary file and cannot display! You may download the full file.");
            List<DomContent> bodyContents = new ArrayList<>();
            if (StringUtils.isNotEmpty(grep)) {
                String matchedString = String.join("\n", Arrays.stream(logString.split("\n")).filter(str -> str.contains(grep)).collect(toList()));
                bodyContents.add(pre(matchedString).withId("logContent"));
            } else {
                DomContent pagerData = null;
                if (isTxtFile(fileName)) {
                    pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "log");
                }
                bodyContents.add(searchFileForm(fileName, "no"));
                // list all files for this topology
                bodyContents.add(logFileSelectionForm(reorderedFilesStr, fileName, "log"));
                if (pagerData != null) {
                    bodyContents.add(pagerData);
                }
                bodyContents.add(downloadLink(fileName));
                bodyContents.add(pre(logString).withClass("logContent"));
                if (pagerData != null) {
                    bodyContents.add(pagerData);
                }
            }
            String content = logTemplate(bodyContents, fileName, user).render();
            return LogviewerResponseBuilder.buildSuccessHtmlResponse(content);
        } else {
            return LogviewerResponseBuilder.buildResponsePageNotFound();
        }
    } else {
        if (resourceAuthorizer.getLogUserGroupWhitelist(fileName) == null) {
            return LogviewerResponseBuilder.buildResponsePageNotFound();
        } else {
            return LogviewerResponseBuilder.buildResponseUnauthorizedUser(user);
        }
    }
}

Example 5 with WorkerLogs

use of org.apache.storm.daemon.logviewer.utils.WorkerLogs in project storm by apache.

the class LogviewerLogPageHandlerTest method createHandlerForTraversalTests.

private LogviewerLogPageHandler createHandlerForTraversalTests(Path rootPath) throws IOException {
    Path daemonLogRoot = rootPath.resolve("logs");
    Path fileOutsideDaemonRoot = rootPath.resolve("evil.sh");
    Path daemonFile = daemonLogRoot.resolve("nimbus.log");
    Path workerLogRoot = daemonLogRoot.resolve("workers-artifacts");
    Path topoA = workerLogRoot.resolve("topoA");
    Path file1 = topoA.resolve("1111").resolve("worker.log");
    Path file2 = topoA.resolve("2222").resolve("worker.log");
    Path file3 = workerLogRoot.resolve("topoB").resolve("1111").resolve("worker.log");
    Files.createDirectories(file1.getParent());
    Files.createDirectories(file2.getParent());
    Files.createDirectories(file3.getParent());
    Files.createFile(file1);
    Files.createFile(file2);
    Files.createFile(file3);
    Files.createFile(fileOutsideDaemonRoot);
    Files.createFile(daemonFile);
    Map<String, Object> stormConf = Utils.readStormConfig();
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    return new LogviewerLogPageHandler(workerLogRoot.toString(), daemonLogRoot.toString(), new WorkerLogs(stormConf, workerLogRoot, metricsRegistry), new ResourceAuthorizer(stormConf), metricsRegistry);
}