Examples with LogNetworkStreamMerger com.emc.storageos.systemservices.impl.logsvc.merger.LogNetworkStreamMerger used on opensource projects

Example 1 with LogNetworkStreamMerger

use of com.emc.storageos.systemservices.impl.logsvc.merger.LogNetworkStreamMerger in project coprhd-controller by CoprHD.

the class SendAlertEvent method generateLogFile.

/**
 * Gets logs with request info provided and populates them to the file path
 * specified, in zip format with fileName as zip entry.
 */
private synchronized void generateLogFile(LogRequest logReqInfo, MediaType mediaType, String filePath, String fileName) throws IOException {
    ZipOutputStream outputStream = null;
    try {
        _log.info("Populating logs to file: {} start", filePath);
        logReqInfo.setMaxBytes(getAttachmentsMaxSizeMB() * BYTE_TO_MB * logSvcPropertiesLoader.getZipFactor());
        final LogNetworkStreamMerger logRequestMgr = new LogNetworkStreamMerger(logReqInfo, mediaType, logSvcPropertiesLoader);
        outputStream = new ZipOutputStream(new FileOutputStream(filePath));
        StreamingOutput responseStream = new StreamingOutput() {

            @Override
            public void write(OutputStream outputStream) {
                try {
                    LogService.runningRequests.incrementAndGet();
                    logRequestMgr.streamLogs(outputStream);
                    _log.info("Total streamed bytes: {}", logRequestMgr.streamedBytes);
                } finally {
                    LogService.runningRequests.decrementAndGet();
                }
            }
        };
        ZipEntry ze = new ZipEntry(fileName);
        outputStream.putNextEntry(ze);
        responseStream.write(outputStream);
        _log.info("Populating logs to file: {} end", filePath);
    } finally {
        if (outputStream != null) {
            outputStream.close();
        }
    }
}

Example 2 with LogNetworkStreamMerger

use of com.emc.storageos.systemservices.impl.logsvc.merger.LogNetworkStreamMerger in project coprhd-controller by CoprHD.

the class LogAnalyser method getNodeErrorLogs.

/*
     * get error logs from the given service
     */
private LogNetworkStreamMerger getNodeErrorLogs() {
    endTime = new DateTime();
    if (startTime == null) {
        startTime = endTime.minusMinutes(15);
    }
    String nodeId = service.getNodeId();
    List<String> nodeIds = new ArrayList<String>();
    nodeIds.add(nodeId);
    LogRequest logReqInfo = new LogRequest.Builder().nodeIds(nodeIds).baseNames(svcNames).logLevel(logLevel).startTime(startTime.toDate()).endTime(endTime.toDate()).regex(msgRegex).maxCont(maxCount).build();
    _log.info("Diagnostics scheduler: log request info is {}", logReqInfo.toString());
    LogNetworkStreamMerger logRequestMgr = new LogNetworkStreamMerger(logReqInfo, MediaType.TEXT_PLAIN_TYPE, logSvcPropertiesLoader);
    // Then it will be start with next log.
    startTime = endTime.plusSeconds(1);
    return logRequestMgr;
}

Example 3 with LogNetworkStreamMerger

use of com.emc.storageos.systemservices.impl.logsvc.merger.LogNetworkStreamMerger in project coprhd-controller by CoprHD.

the class LogAnalyser method analysisLogs.

public void analysisLogs() {
    // parse db and zk error logs and alert if match pre-defined errors/fatals
    try {
        String serviceNameList = getServiceNameList();
        _log.info("Starting parse error logs for services : {}, and will alert if match pre-defined errors/fatals", serviceNameList);
        LogNetworkStreamMerger logRequestMgr = getNodeErrorLogs();
        LogMessage msg = logRequestMgr.readNextMergedLogMessage();
        int finalCount = 0;
        if (msg != null) {
            do {
                List<LogMessage> currentLogBatch = new ArrayList<>();
                LogMessage startLogOfNextBatch = logRequestMgr.readLogBatch(msg, currentLogBatch);
                if (!LogUtil.permitNextLogBatch(maxCount, finalCount, currentLogBatch.size())) {
                    // discard this batch
                    break;
                }
                for (LogMessage logMsg : currentLogBatch) {
                    parseErrorLogAndCompareWithPatterns(logMsg);
                    finalCount++;
                }
                msg = startLogOfNextBatch;
            } while (msg != null);
        }
        _log.info("Total error/fatal logs number is {}", finalCount);
    } catch (Exception e) {
        _log.error("Get exception when achieve logs with error msg: {}; stack trace is {}", e.getMessage(), e.getStackTrace());
    }
}

Example 4 with LogNetworkStreamMerger

use of com.emc.storageos.systemservices.impl.logsvc.merger.LogNetworkStreamMerger in project coprhd-controller by CoprHD.

the class LogService method getLogs.

/**
 * Get log data from the specified virtual machines that are filtered, merged,
 * and sorted based on the passed request parameters and streams the log
 * messages back to the client as JSON formatted strings.
 *
 * @brief Show logs from all or specified virtual machine
 * @param nodeIds The ids of the virtual machines for which log data is
 *            collected.
 *            Allowed values: standalone,
 *            control nodes: vipr1,vipr2 etc
 *            data services nodes: dataservice-10-111-111-222 (node-ip-address)
 * @param nodeNames The custom names of the vipr nodes for which log data is
 *            collected.
 *            Allowed values: Current values of node_x_name properties
 * @param logNames The names of the log files to process.
 * @param severity The minimum severity level for a logged message.
 *            Allowed values:0-9. Default value: 7
 * @param startTimeStr The start datetime of the desired time window. Value is
 *            inclusive.
 *            Allowed values: "yyyy-MM-dd_HH:mm:ss" formatted date or
 *            datetime in ms.
 *            Default: Set to yesterday same time
 * @param endTimeStr The end datetime of the desired time window. Value is
 *            inclusive.
 *            Allowed values: "yyyy-MM-dd_HH:mm:ss" formatted date or
 *            datetime in ms.
 * @param msgRegex A regular expression to which the log message conforms.
 * @param maxCount Maximum number of log messages to retrieve. This may return
 *            more than max count, if there are more messages with same
 *            timestamp as of the latest message.
 *            Value should be greater than 0.
 * @param dryRun if true, the API will do a dry run for log collection. Instead
 *            of collecting logs from nodes, dry run will check the nodes'
 *            availability for collecting logs. Entity body of the response
 *            will return an error message string indicating which node(s)
 *            not available for collecting logs. If log collection is ok
 *            for all specified nodes, no error message is included in
 *            response.
 *            Default value of this parameter is false.
 * @prereq none
 * @return A reference to the StreamingOutput to which the log data is
 *         written.
 * @throws WebApplicationException When an invalid request is made.
 */
@GET
@CheckPermission(roles = { Role.SYSTEM_ADMIN, Role.SYSTEM_MONITOR, Role.SECURITY_ADMIN })
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN })
public Response getLogs(@QueryParam(LogRequestParam.NODE_ID) List<String> nodeIds, @QueryParam(LogRequestParam.NODE_NAME) List<String> nodeNames, @QueryParam(LogRequestParam.LOG_NAME) List<String> logNames, @DefaultValue(LogSeverity.DEFAULT_VALUE_AS_STR) @QueryParam(LogRequestParam.SEVERITY) int severity, @QueryParam(LogRequestParam.START_TIME) String startTimeStr, @QueryParam(LogRequestParam.END_TIME) String endTimeStr, @QueryParam(LogRequestParam.MSG_REGEX) String msgRegex, @QueryParam(LogRequestParam.MAX_COUNT) int maxCount, @QueryParam(LogRequestParam.DRY_RUN) @DefaultValue("false") boolean dryRun) throws Exception {
    _log.info("Received getlogs request");
    enforceRunningRequestLimit();
    final MediaType mediaType = getMediaType();
    _log.info("Logs request media type {}", mediaType);
    nodeIds = _coordinatorClientExt.combineNodeNamesWithNodeIds(nodeNames, nodeIds);
    // Validate the passed node ids.
    validateNodeIds(nodeIds);
    _log.debug("Validated requested nodes");
    // Validate the passed severity is valid.
    validateLogSeverity(severity);
    _log.debug("Validated requested severity");
    // Validate the passed start and end times are valid.
    Date startTime = TimeUtils.getDateTimestamp(startTimeStr);
    Date endTime = TimeUtils.getDateTimestamp(endTimeStr);
    TimeUtils.validateTimestamps(startTime, endTime);
    _log.debug("Validated requested time window");
    // Setting default start time to yesterday
    if (startTime == null) {
        Calendar yesterday = Calendar.getInstance();
        yesterday.add(Calendar.DATE, -1);
        startTime = yesterday.getTime();
        _log.info("Setting start time to yesterday {} ", startTime);
    }
    // Validate regular message
    validateMsgRegex(msgRegex);
    _log.debug("Validated regex");
    // Validate max count
    if (maxCount < 0) {
        throw APIException.badRequests.parameterIsNotValid("maxCount");
    }
    // validate log names
    Set<String> allLogNames = getValidLogNames();
    _log.debug("valid log names {}", allLogNames);
    boolean invalidLogName = false;
    for (String logName : logNames) {
        if (!allLogNames.contains(logName)) {
            invalidLogName = true;
            break;
        }
    }
    if (invalidLogName) {
        throw APIException.badRequests.parameterIsNotValid("log names");
    }
    if (dryRun) {
        List<NodeInfo> clusterNodesInfo = ClusterNodesUtil.getClusterNodeInfo();
        if (clusterNodesInfo.isEmpty()) {
            _log.error("No nodes available for collecting logs");
            throw APIException.internalServerErrors.noNodeAvailableError("no nodes available for collecting logs");
        }
        List<NodeInfo> matchingNodes = null;
        if (nodeIds.isEmpty()) {
            matchingNodes = clusterNodesInfo;
        } else {
            matchingNodes = new ArrayList<NodeInfo>();
            for (NodeInfo node : clusterNodesInfo) {
                if (nodeIds.contains(node.getId())) {
                    matchingNodes.add(node);
                }
            }
        }
        // find the unavailable nodes
        List<String> failedNodes = null;
        if (matchingNodes.size() == 1 && matchingNodes.get(0).getId().equals("standalone")) {
            failedNodes = new ArrayList<String>();
        } else {
            // find the unavailable nodes
            failedNodes = _coordinatorClientExt.getUnavailableControllerNodes();
        }
        if (!nodeIds.isEmpty()) {
            failedNodes.retainAll(nodeIds);
        }
        String baseNodeURL;
        SysClientFactory.SysClient sysClient;
        for (final NodeInfo node : matchingNodes) {
            baseNodeURL = String.format(SysClientFactory.BASE_URL_FORMAT, node.getIpAddress(), node.getPort());
            _log.debug("getting log names from node: " + baseNodeURL);
            sysClient = SysClientFactory.getSysClient(URI.create(baseNodeURL), _logSvcPropertiesLoader.getNodeLogCollectorTimeout() * 1000, _logSvcPropertiesLoader.getNodeLogConnectionTimeout() * 1000);
            LogRequest logReq = new LogRequest.Builder().nodeIds(nodeIds).baseNames(getLogNamesFromAlias(logNames)).logLevel(severity).startTime(startTime).endTime(endTime).regex(msgRegex).maxCont(maxCount).build();
            logReq.setDryRun(true);
            try {
                sysClient.post(SysClientFactory.URI_NODE_LOGS, null, logReq);
            } catch (Exception e) {
                _log.error("Exception accessing node {}: {}", baseNodeURL, e);
                failedNodes.add(node.getId());
            }
        }
        if (_coordinatorClientExt.getNodeCount() == failedNodes.size()) {
            throw APIException.internalServerErrors.noNodeAvailableError("All nodes are unavailable for collecting logs");
        }
        return Response.ok().build();
    }
    LogRequest logReqInfo = new LogRequest.Builder().nodeIds(nodeIds).baseNames(getLogNamesFromAlias(logNames)).logLevel(severity).startTime(startTime).endTime(endTime).regex(msgRegex).maxCont(maxCount).build();
    _log.info("log request info is {}", logReqInfo.toString());
    final LogNetworkStreamMerger logRequestMgr = new LogNetworkStreamMerger(logReqInfo, mediaType, _logSvcPropertiesLoader);
    StreamingOutput logMsgStream = new StreamingOutput() {

        @Override
        public void write(OutputStream outputStream) {
            try {
                runningRequests.incrementAndGet();
                logRequestMgr.streamLogs(outputStream);
            } finally {
                runningRequests.decrementAndGet();
            }
        }
    };
    return Response.ok(logMsgStream).build();
}