Search in sources :

Example 1 with Unchecked

use of org.jooq.lambda.Unchecked in project storm by apache.

the class LogviewerLogSearchHandler method deepSearchLogsForTopology.

/**
 * Advanced search across worker log files in a topology.
 *
 * @param topologyId topology ID
 * @param user username
 * @param search search string
 * @param numMatchesStr the count of maximum matches. Note that this number is with respect to each port, not to each log or each search
 *     request
 * @param portStr worker port, null or '*' if the request wants to search from all worker logs
 * @param fileOffsetStr index (offset) of the log files
 * @param offsetStr start offset for log file
 * @param searchArchived true if the request wants to search also archived files, false if not
 * @param callback callbackParameterName for JSONP
 * @param origin origin
 * @return Response containing JSON content representing search result
 */
public Response deepSearchLogsForTopology(String topologyId, String user, String search, String numMatchesStr, String portStr, String fileOffsetStr, String offsetStr, Boolean searchArchived, String callback, String origin) throws IOException {
    int numMatchedFiles = 0;
    int numScannedFiles = 0;
    Path rootDir = logRoot;
    Path absTopoDir = rootDir.resolve(topologyId).toAbsolutePath().normalize();
    Object returnValue;
    if (StringUtils.isEmpty(search) || !absTopoDir.toFile().exists() || !absTopoDir.startsWith(rootDir)) {
        returnValue = new ArrayList<>();
    } else {
        int fileOffset = ObjectReader.getInt(fileOffsetStr, 0);
        int offset = ObjectReader.getInt(offsetStr, 0);
        int numMatches = ObjectReader.getInt(numMatchesStr, 1);
        if (StringUtils.isEmpty(portStr) || portStr.equals("*")) {
            try (Stream<Path> topoDir = Files.list(absTopoDir)) {
                // check for all ports
                Stream<List<Path>> portsOfLogs = topoDir.map(portDir -> logsForPort(user, portDir)).filter(logs -> logs != null && !logs.isEmpty());
                if (BooleanUtils.isNotTrue(searchArchived)) {
                    portsOfLogs = portsOfLogs.map(fl -> Collections.singletonList(first(fl)));
                }
                final List<Matched> matchedList = portsOfLogs.map(logs -> findNMatches(logs, numMatches, 0, 0, search)).collect(toList());
                numMatchedFiles = matchedList.stream().mapToInt(match -> match.getMatches().size()).sum();
                numScannedFiles = matchedList.stream().mapToInt(match -> match.openedFiles).sum();
                returnValue = matchedList;
            }
        } else {
            int port = Integer.parseInt(portStr);
            // check just the one port
            @SuppressWarnings("unchecked") List<Integer> slotsPorts = SupervisorUtils.getSlotsPorts(stormConf);
            boolean containsPort = slotsPorts.stream().anyMatch(slotPort -> slotPort != null && (slotPort == port));
            if (!containsPort) {
                returnValue = new ArrayList<>();
            } else {
                Path absPortDir = absTopoDir.resolve(Integer.toString(port)).toAbsolutePath().normalize();
                if (!absPortDir.toFile().exists() || !absPortDir.startsWith(absTopoDir)) {
                    returnValue = new ArrayList<>();
                } else {
                    List<Path> filteredLogs = logsForPort(user, absPortDir);
                    if (BooleanUtils.isNotTrue(searchArchived)) {
                        filteredLogs = Collections.singletonList(first(filteredLogs));
                        fileOffset = 0;
                    }
                    returnValue = findNMatches(filteredLogs, numMatches, fileOffset, offset, search);
                    numMatchedFiles = ((Matched) returnValue).getMatches().size();
                    numScannedFiles = ((Matched) returnValue).openedFiles;
                }
            }
        }
    }
    if (numMatchedFiles == 0) {
        numDeepSearchNoResult.mark();
    }
    numFileScanned.update(numScannedFiles);
    return LogviewerResponseBuilder.buildSuccessJsonResponse(returnValue, callback, origin);
}
Also used : Path(java.nio.file.Path) Arrays(java.util.Arrays) GZIPInputStream(java.util.zip.GZIPInputStream) StringUtils(org.apache.commons.lang.StringUtils) BufferedInputStream(java.io.BufferedInputStream) PathUtil.truncatePathToLastElements(org.apache.storm.daemon.utils.PathUtil.truncatePathToLastElements) ListFunctionalSupport.rest(org.apache.storm.daemon.utils.ListFunctionalSupport.rest) LoggerFactory(org.slf4j.LoggerFactory) FileTime(java.nio.file.attribute.FileTime) LogviewerConstant(org.apache.storm.daemon.logviewer.LogviewerConstant) ByteBuffer(java.nio.ByteBuffer) BooleanUtils(org.apache.commons.lang.BooleanUtils) Pair(org.apache.commons.lang3.tuple.Pair) DaemonConfig(org.apache.storm.DaemonConfig) Map(java.util.Map) ListFunctionalSupport.drop(org.apache.storm.daemon.utils.ListFunctionalSupport.drop) ExceptionMeterNames(org.apache.storm.daemon.logviewer.utils.ExceptionMeterNames) Path(java.nio.file.Path) UrlBuilder(org.apache.storm.daemon.utils.UrlBuilder) Unchecked(org.jooq.lambda.Unchecked) LogviewerResponseBuilder(org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder) StandardCharsets(java.nio.charset.StandardCharsets) List(java.util.List) Stream(java.util.stream.Stream) Response(javax.ws.rs.core.Response) Pattern(java.util.regex.Pattern) UnsupportedEncodingException(java.io.UnsupportedEncodingException) InvalidRequestException(org.apache.storm.daemon.ui.InvalidRequestException) Histogram(com.codahale.metrics.Histogram) ListFunctionalSupport.last(org.apache.storm.daemon.utils.ListFunctionalSupport.last) JSONAware(org.json.simple.JSONAware) HashMap(java.util.HashMap) SupervisorUtils(org.apache.storm.daemon.supervisor.SupervisorUtils) ArrayList(java.util.ArrayList) Meter(com.codahale.metrics.Meter) DirectoryCleaner(org.apache.storm.daemon.logviewer.utils.DirectoryCleaner) ServerUtils(org.apache.storm.utils.ServerUtils) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) ResourceAuthorizer(org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) Logger(org.slf4j.Logger) ListFunctionalSupport.first(org.apache.storm.daemon.utils.ListFunctionalSupport.first) Files(java.nio.file.Files) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) Utils(org.apache.storm.utils.Utils) UnknownHostException(java.net.UnknownHostException) WorkerLogs(org.apache.storm.daemon.logviewer.utils.WorkerLogs) StreamUtil(org.apache.storm.daemon.utils.StreamUtil) Collectors.toList(java.util.stream.Collectors.toList) ObjectReader(org.apache.storm.utils.ObjectReader) JsonResponseBuilder(org.apache.storm.daemon.common.JsonResponseBuilder) Paths(java.nio.file.Paths) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Collections(java.util.Collections) InputStream(java.io.InputStream) List(java.util.List) ArrayList(java.util.ArrayList) Collectors.toList(java.util.stream.Collectors.toList)

Aggregations

Histogram (com.codahale.metrics.Histogram)1 Meter (com.codahale.metrics.Meter)1 JsonIgnore (com.fasterxml.jackson.annotation.JsonIgnore)1 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 BufferedInputStream (java.io.BufferedInputStream)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 UnsupportedEncodingException (java.io.UnsupportedEncodingException)1 UnknownHostException (java.net.UnknownHostException)1 ByteBuffer (java.nio.ByteBuffer)1 StandardCharsets (java.nio.charset.StandardCharsets)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 Paths (java.nio.file.Paths)1 FileTime (java.nio.file.attribute.FileTime)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1