Examples with LogPatternInfo com.creditease.agent.feature.logagent.objects.LogPatternInfo used on opensource projects

Example 1 with LogPatternInfo

use of com.creditease.agent.feature.logagent.objects.LogPatternInfo in project uavstack by uavorg.

the class LogAgent method notifyProfileDataUpdate.

/**
 * notify there is profile data update
 *
 * @param profileData
 */
@SuppressWarnings("rawtypes")
private void notifyProfileDataUpdate(MonitorDataFrame profileData, JVMAgentInfo agentInfo) {
    /**
     * Step 1: parse MDF to build NewLogPatternInfoMap
     */
    AppLogPatternInfoCollection newLogPatternInfoMap = new AppLogPatternInfoCollection();
    Map<String, LogPatternInfo> newLogPatternPathConflictMap = new HashMap<String, LogPatternInfo>();
    // jvm system properties for ${xxxx} varable
    Properties systemPro = agentInfo.getSystemProperties();
    for (Entry<String, List<Map>> e : profileData.getDatas().entrySet()) {
        // get appid
        String appID = e.getKey();
        // get webapproot
        Map<String, Object> webappInfo = profileData.getElemInstValues(appID, "cpt", "webapp");
        String webAppRoot = (String) webappInfo.get("webapproot");
        if (null == webAppRoot) {
            // notify
            String content = "Application[" + appID + "]'s profileData is incomplete: field[webapproot] is null.";
            log.warn(this, content);
            NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogAppProfileError, "AppProfileError", content);
            event.addArg("serverid", profileData.getServerId());
            event.addArg("appid", appID);
            this.putNotificationEvent(event);
            continue;
        }
        String appurl = (String) webappInfo.get("appurl");
        // get PEId=logs
        List<Map> logsPD = profileData.getElemInstances(appID, "logs");
        for (Map logPD : logsPD) {
            Map logInstanceValues = (Map) logPD.get("values");
            for (Object keyObj : logInstanceValues.keySet()) {
                String logPattern = (String) keyObj;
                // figure out absolute path
                String absPath = figureOutAbsolutePath(webAppRoot, logPattern, systemPro);
                LogPatternInfo lpi = new LogPatternInfo();
                // set appid
                lpi.setAppId(appID);
                // set servid
                lpi.setServId(profileData.getServerId());
                // set logpattern
                lpi.setLogParttern(logPattern);
                // set webapproot
                lpi.setWebAppRoot(webAppRoot);
                // set appurl
                lpi.setAppUrl(appurl);
                // set abspath
                lpi.setAbsolutePath(absPath);
                // set state flag as NEWCOME
                lpi.setFlag(StateFlag.NEWCOME);
                // set timeStamp as current
                lpi.setTimeStamp(System.currentTimeMillis());
                // Step 1.1: check if the logpattern exists in ISSUE MAP
                LogPatternInfo ilpi = this.IssueLogProfileDataMap.get(lpi.getAppUUID(), lpi.getUUID());
                // project may have same serverID,appID,logpattern,but has different absolutepath -- by hongqiangwei
                if (null != ilpi) {
                    // nothing
                    if (ilpi.getAbsolutePath().equalsIgnoreCase(lpi.getAbsolutePath())) {
                        continue;
                    } else // if there is change, then remove it from ISSUE MAP,
                    // this logpattern may go to NEW MAP
                    {
                        this.IssueLogProfileDataMap.remove(lpi.getAppUUID(), lpi.getUUID());
                    }
                }
                // Step 1.2: we should check if there is conflict among the
                // new coming profile data
                boolean isDuplicated = newLogPatternPathConflictMap.containsKey(lpi.getAbsolutePath());
                /**
                 * if duplicated, that means this logpattern has issue and just put it into IssueLogProfileDataMap
                 */
                if (isDuplicated == true) {
                    // move to ISSUE MAP
                    LogPatternInfo templpi = newLogPatternPathConflictMap.get(lpi.getAbsolutePath());
                    LogPatternInfo originlpi = LatestLogProfileDataMap.get(lpi.getAppUUID(), lpi.getUUID());
                    if (originlpi != null) {
                        // remove new(who is not exist in
                        // LatestLogProfileDataMap) from
                        // newLogPatternInfoMap and
                        // newLogPatternPathConflictMap
                        newLogPatternInfoMap.remove(templpi.getAppUUID(), templpi.getUUID());
                        newLogPatternPathConflictMap.remove(templpi.getAbsolutePath());
                        // add new(who is not exist in
                        // LatestLogProfileDataMap) to
                        // IssueLogProfileDataMap
                        this.IssueLogProfileDataMap.put(templpi);
                        // add current(who is exist in
                        // LatestLogProfileDataMap) to newLogPatternInfoMap
                        // and newLogPatternPathConflictMap
                        newLogPatternInfoMap.put(lpi);
                        newLogPatternPathConflictMap.put(lpi.getAbsolutePath(), lpi);
                    } else {
                        this.IssueLogProfileDataMap.put(lpi);
                    }
                    log.err(this, "APPUUID : " + lpi.getAppUUID() + " whoese log absolute path conflict with APPUUID :" + templpi.getAppUUID());
                    String content = "APPUUID : " + lpi.getAppUUID() + " whoese log absolute path conflict with APPUUID :" + templpi.getAppUUID();
                    NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogPathConflict, "LogPathConflict", content);
                    this.putNotificationEvent(event);
                } else /**
                 * if not duplicated, that means this logpattern seems NEWCOME just at this moment.
                 */
                {
                    newLogPatternPathConflictMap.put(lpi.getAbsolutePath(), lpi);
                    // move to NEW MAP
                    newLogPatternInfoMap.put(lpi);
                }
            }
        }
    }
    /**
     * Step 2: Compare newLogPatternInfoMap with LatestLogProfileDataMap to find NEWCOME or UPDATE
     */
    for (Entry<String, Map<String, LogPatternInfo>> entry : newLogPatternInfoMap.entrySet()) {
        String appUUID = entry.getKey();
        Map<String, LogPatternInfo> logPatternMap = entry.getValue();
        boolean lC = this.LatestLogProfileDataMap.containsKey(appUUID);
        for (LogPatternInfo lpi : logPatternMap.values()) {
            // appUUID EXISTs in LatestLogProfileDataMap
            if (lC == true) {
                LogPatternInfo ilpi = this.LatestLogProfileDataMap.get(appUUID, lpi.getUUID());
                // the state should be UPDATE
                if (null != ilpi) {
                    if (!lpi.getAbsolutePath().equals(ilpi.getAbsolutePath())) {
                        lpi.setFlag(StateFlag.UPDATE);
                    } else {
                        lpi.setFlag(StateFlag.EXIST);
                    }
                }
            }
            // add this logpattern to LatestLogProfileDataMap
            // NOTE: currently we are ignoring the abspath's conflicts in
            // LatestLogProfileDataMap,
            // we will handle that in next step
            this.LatestLogProfileDataMap.put(lpi);
        }
    }
    /**
     * Step 3: Compare updated LatestLogProfileDataMap with newLogPatternInfoMap again to find EXIST_UNKOWN
     * logpattern
     */
    for (Entry<String, Map<String, LogPatternInfo>> entry : LatestLogProfileDataMap.entrySet()) {
        String appUUID = entry.getKey();
        Map<String, LogPatternInfo> logPatternMap = entry.getValue();
        boolean isInNew = newLogPatternInfoMap.containsKey(appUUID);
        for (LogPatternInfo lpi : logPatternMap.values()) {
            // appUUID in NEWCOME
            if (isInNew) {
                // check if the logpattern exists in NEWCOME
                LogPatternInfo ilpi = newLogPatternInfoMap.get(appUUID, lpi.getUUID());
                // logUUID not in NEWCOME
                if (ilpi == null) {
                    // mark this logpattern in EXIST_UNKOWN,logCatcher
                    // should fix the state
                    lpi.setFlag(StateFlag.EXIST_UNKOWN);
                }
            } else // appUUID not in NEWCOME
            {
                // mark this logpattern in EXIST_UNKOWN,logCatcher should
                // fix the state
                lpi.setFlag(StateFlag.EXIST_UNKOWN);
            }
        }
    }
    /**
     * Step 4: so sorry, we should rescan the LatestLogProfileDataMap, currently only EXIST_UNKOWN,UPDATE,NEWCOME
     * left to ensure the following conflict rules: 1) UPATE should kick off NEWCOME: 必然存在 2) EXIST_UNKOWN should
     * kick off NEWCOME：尽管可能EXIST_UNKOWN不存在，但保持现状吧，需要logCatcher进行修正, 如果EXIST_UNKOWN存在， 则更新为EXIST，则NEWCOME继续保持在ISSUE
     * MAP 如果EXIST_UNKOWN不存在，则将之移动到ISSUE MAP;其他所有NEWCOME继续保持在ISSUE MAP
     */
    Map<String, List<LogPatternInfo>> logPatternInfoToIssuesMap = new HashMap<String, List<LogPatternInfo>>();
    for (Entry<String, Map<String, LogPatternInfo>> entry : LatestLogProfileDataMap.entrySet()) {
        Map<String, LogPatternInfo> logPatternMap = entry.getValue();
        for (LogPatternInfo lpi : logPatternMap.values()) {
            String absPath = lpi.getAbsolutePath();
            if (!logPatternInfoToIssuesMap.containsKey(absPath)) {
                List<LogPatternInfo> lpiList = new ArrayList<LogPatternInfo>();
                lpiList.add(lpi);
                logPatternInfoToIssuesMap.put(absPath, lpiList);
            } else {
                List<LogPatternInfo> lpiList = logPatternInfoToIssuesMap.get(absPath);
                lpiList.add(lpi);
                /**
                 * sort the list, then every list for one application should have order by flag weight
                 */
                Collections.sort(lpiList, new Comparator<LogPatternInfo>() {

                    @Override
                    public int compare(LogPatternInfo o1, LogPatternInfo o2) {
                        int weightSpan = o2.getFlag().getFlagWeight() - o1.getFlag().getFlagWeight();
                        return weightSpan;
                    }
                });
            }
        }
    }
    /**
     * Step 5: build logCatcherInfoMap
     */
    Map<String, LogPatternInfo> logCatcherInfoMap = new LinkedHashMap<String, LogPatternInfo>();
    for (List<LogPatternInfo> lpiList : logPatternInfoToIssuesMap.values()) {
        LogPatternInfo lpi = lpiList.get(0);
        // we just need the first one as the updated one
        logCatcherInfoMap.put(lpi.getUUID(), lpi);
        // the other ones go to ISSUE MAP
        for (int i = 1; i < lpiList.size(); i++) {
            this.IssueLogProfileDataMap.put(lpiList.get(i));
            LatestLogProfileDataMap.remove(lpiList.get(i).getAppUUID(), lpiList.get(i).getUUID());
        }
    }
    /**
     * in some cases there are may no log pattern infos
     */
    if (logCatcherInfoMap.isEmpty()) {
        log.warn(this, "The logCatcherInfoMap is empty and will not update log pattern info in ProfileData for logCatcher.");
        return;
    }
    /**
     * start logCatcher
     *
     * logCatcher is used to process logs
     */
    if (null == logCatcher) {
        String positionfileroot = this.getConfigManager().getContext(IConfigurationManager.METADATAPATH);
        logCatcher = new TaildirLogComponent("TaildirlogComponent", this.feature);
        // support multiple file in future,add dynamic update tailfiles list
        ConfigurationContext context = new ConfigurationContext();
        context.put(POSITION_FILE_ROOT, positionfileroot);
        try {
            logCatcher.configure(context, logCatcherInfoMap);
            logCatcher.start();
            log.info(this, "ApplicationServer LogCatcher starts SUCCESS");
        } catch (IOException e) {
            log.err(this, "ApplicationServer LogCatcher starts FAIL.", e);
            return;
        }
    } else /**
     * update logCatcher
     */
    {
        logCatcher.configure(logCatcherInfoMap);
        log.info(this, "ApplicationServer LogCatcher updates SUCCESS");
    }
}

Example 2 with LogPatternInfo

use of com.creditease.agent.feature.logagent.objects.LogPatternInfo in project uavstack by uavorg.

the class TaildirLogComponent method tailFileCommon.

@SuppressWarnings("rawtypes")
public void tailFileCommon(TailFile tf, boolean backoffWithoutNL, Map<TailFile, List<Map>> serverlogs) throws IOException, InterruptedException {
    long current = System.currentTimeMillis();
    boolean isEvents = false;
    // while (true) {
    reader.setCurrentFile(tf);
    List<Event> events = reader.readEvents(batchSize, backoffWithoutNL);
    if (!events.isEmpty()) {
        isEvents = true;
    }
    try {
        LogFilterAndRule main = RuleFilterFactory.getInstance().getLogFilterAndRule(tf.getPath());
        List<LogFilterAndRule> aids = RuleFilterFactory.getInstance().getAidLogFilterAndRuleList(tf.getPath());
        List<Map> datalog = RuleFilterFactory.getInstance().createChain(reader, batchSize).setMainLogFilterAndRule(main).setAidLogFilterAndRuleList(aids).doProcess(events, backoffWithoutNL);
        if (!datalog.isEmpty()) {
            if (serverlogs.containsKey(tf)) {
                serverlogs.get(tf).addAll(datalog);
            } else
                serverlogs.put(tf, datalog);
        }
        reader.commit(events.size() < batchSize);
    } catch (IOException ex) {
        log.warn(this, "The unexpected failure. " + "The source will try again after " + retryInterval + " ms");
    // TimeUnit.MILLISECONDS.sleep(retryInterval);
    // retryInterval = retryInterval << 1;
    // retryInter val = Math.min(retryInterval, maxRetryInterval);
    // continue;
    }
    // retryInterval = 1000;
    // if (events.size() < batchSize) {
    // break;
    // }
    // }
    // renew
    LogAgent logagent = (LogAgent) ConfigurationManager.getInstance().getComponent("logagent", "LogAgent");
    LogPatternInfo info = logagent.getLatestLogProfileDataMap().get(tf.getServerId() + "-" + tf.getAppId(), tf.getId());
    if (info != null && isEvents) {
        info.setTimeStamp(current);
        LogPatternInfo innerInfo = reader.getTailFileTable().asMap().get(info.getAbsolutePath());
        if (innerInfo != null) {
            innerInfo.setTimeStamp(current);
            // FIXME concurrent problem
            reader.getTailFileTable().put(innerInfo.getAbsolutePath(), innerInfo);
        }
    }
    if (info != null && current - info.getTimeStamp() > timeOutInterval) {
        logagent.getLatestLogProfileDataMap().remove(tf.getServerId() + "-" + tf.getAppId(), tf.getId());
        // notify
        String title = NetworkHelper.getLocalIP() + "日志[" + tf.getId() + "]的过滤规则配置已经过期.";
        log.err(this, title);
        NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogRuleExpired, title, title);
        event.addArg("serverid", tf.getServerId());
        event.addArg("appid", tf.getAppId());
        this.putNotificationEvent(event);
    }
}

Example 3 with LogPatternInfo

use of com.creditease.agent.feature.logagent.objects.LogPatternInfo in project uavstack by uavorg.

the class LogAgent method updateAllStrategy.

@SuppressWarnings({ "rawtypes", "unchecked" })
private void updateAllStrategy(String stragetyJson) {
    // full stragety
    List<Map> list = JSONHelper.toObjectArray(stragetyJson, Map.class);
    if (list == null || list.size() == 0) {
        log.warn(this, "stragety json is empty!");
        return;
    }
    for (Map m : list) {
        String serverid = (String) m.get("servid");
        String appid = (String) m.get("appid");
        String logid = (String) m.get("logid");
        String filter = (String) m.get("filter");
        String separator = (String) m.get("separator");
        String fields = (String) m.get("fields");
        LogPatternInfo lcfg = new LogPatternInfo();
        lcfg.setServId(serverid);
        lcfg.setAppId(appid);
        lcfg.setLogParttern(logid);
        LogPatternInfo info = this.LatestLogProfileDataMap.get(lcfg.getAppUUID(), lcfg.getUUID());
        if (info == null) {
            log.warn(this, "log pattern info is null! loguuid:" + lcfg.getUUID());
            return;
        }
        String logPath = info.getAbsolutePath();
        Map mapping = new HashMap();
        mapping.putAll(m);
        mapping.put("absPath", logPath);
        logCfgMapping.put(lcfg.getUUID(), mapping);
        LogFilterAndRule lfar = new DefaultLogFilterAndRule(filter, separator, JSON.parseObject(fields), 0, 0);
        RuleFilterFactory.getInstance().pubLogFilterAndRule(logPath, lfar);
    }
    saveLogCfg();
}

Example 4 with LogPatternInfo

use of com.creditease.agent.feature.logagent.objects.LogPatternInfo in project uavstack by uavorg.

the class ReliableTaildirEventReader method updateTailFiles.

/**
 * Update tailFiles mapping if a new file is created or appends are detected to the existing file.
 */
public List<Long> updateTailFiles(boolean skipToEnd) throws IOException {
    LogAgent logagent = (LogAgent) ConfigurationManager.getInstance().getComponent("logagent", "LogAgent");
    updateTime = System.currentTimeMillis();
    List<Long> updatedInodes = Lists.newArrayList();
    String serverid = null;
    String appid = null;
    String logid = null;
    String appurl = null;
    for (Entry<String, LogPatternInfo> cell : tailFileTable.asMap().entrySet()) {
        // cell<serverid--appid--logid, logpath, logname>
        // 
        Map<String, String> headers = headerTable.row(cell.getKey());
        LogPatternInfo logPatternInfo = cell.getValue();
        // 文件父路径
        File parentDir = logPatternInfo.getParentDir();
        // 编译后的文件名
        Pattern fileNamePattern = logPatternInfo.getLogRegxPattern();
        serverid = logPatternInfo.getServId();
        appid = logPatternInfo.getAppId();
        logid = logPatternInfo.getLogParttern();
        appurl = logPatternInfo.getAppUrl();
        List<File> files = getMatchFiles(parentDir, fileNamePattern);
        LogPatternInfo logPatternInfo2 = logagent.getLatestLogProfileDataMap().get(logPatternInfo.getAppUUID(), logPatternInfo.getUUID());
        if (!files.isEmpty()) {
            // modify status UNKNOWN to EXISTS
            if (logPatternInfo2 != null) {
                logPatternInfo2.setFlag(StateFlag.EXIST);
            }
        } else if (logPatternInfo2.getFlag() == StateFlag.EXIST) {
            logPatternInfo2.setFlag(StateFlag.EXIST_UNKOWN);
            String title = NetworkHelper.getLocalIP() + "曾经在" + logPatternInfo.getParentDir() + "符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]的日志文件消失了。";
            String content = "失败原因：1）错误删除了这些日志文件。2）修改了日志文件名称，且新名称不符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]。";
            logger.warn(this, title);
            NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogNotExist, title, content);
            event.addArg("serverid", logPatternInfo.getServId());
            event.addArg("appid", logPatternInfo.getAppId());
            logagent.putNotificationEvent(event);
        }
        for (File f : files) {
            long inode = getInode(f);
            TailFile tf = tailFiles.get(inode);
            if (tf == null || !tf.getPath().equals(f.getAbsolutePath())) {
                // 第一次读取从头开始读
                long startPos = skipToEnd ? f.length() : 0;
                // how to get line's number ?
                long startNum = 0;
                // try to get pos form position file
                if (maybeReloadMap.containsKey(inode)) {
                    startPos = maybeReloadMap.get(inode)[0];
                    startNum = maybeReloadMap.get(inode)[1];
                }
                tf = openFile(serverid, appid, logid, f, headers, inode, startPos, startNum);
                tf.setAppUrl(appurl);
            } else {
                boolean updated = tf.getLastUpdated() < f.lastModified();
                if (updated) {
                    if (tf.getRaf() == null) {
                        // 获取文件的读取手柄
                        tf = openFile(serverid, appid, logid, f, headers, inode, tf.getPos(), tf.getNum());
                        tf.setAppUrl(appurl);
                    }
                    if (f.length() < tf.getPos()) {
                        // 文件的长度小于上次读取的指针说明文件内容被删除了，改成从0读取
                        logger.info(this, "Pos " + tf.getPos() + " is larger than file size! " + "Restarting from pos 0, file: " + tf.getPath() + ", inode: " + inode);
                        tf.updatePos(tf.getPath(), inode, 0, 0);
                    }
                }
                // 设置是否需要监控指标
                tf.setNeedTail(updated);
            }
            tailFiles.put(inode, tf);
            updatedInodes.add(inode);
            if (logger.isDebugEnable()) {
                logger.debug(this, "tailfile mapping: " + inode + " --> " + tf.getId());
            }
        }
    }
    return updatedInodes;
}

Example 5 with LogPatternInfo

use of com.creditease.agent.feature.logagent.objects.LogPatternInfo in project uavstack by uavorg.

the class ReliableTaildirEventReader method updatelog.

public void updatelog(Map<String, LogPatternInfo> filePaths) {
    for (Entry<String, LogPatternInfo> e : filePaths.entrySet()) {
        LogPatternInfo logPatternInfo = e.getValue();
        LogAgent logagent = (LogAgent) ConfigurationManager.getInstance().getComponent("logagent", "LogAgent");
        LogPatternInfo logPatternInfoTemp = logagent.getLatestLogProfileDataMap().get(logPatternInfo.getAppUUID(), logPatternInfo.getUUID());
        List<File> list = getMatchFiles(logPatternInfo.getParentDir(), logPatternInfo.getLogRegxPattern());
        if (!list.isEmpty()) {
            logPatternInfoTemp.setFlag(StateFlag.EXIST);
        } else {
            logPatternInfoTemp.setFlag(StateFlag.EXIST_UNKOWN);
        }
        if (list.isEmpty() && !logPatternInfo.getParentDir().isDirectory()) {
            // notify
            String title = NetworkHelper.getLocalIP() + "在" + logPatternInfo.getParentDir() + "下没有符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]的日志文件。";
            String content = "失败原因：1）日志文件匹配规则配置错误，所以不能定位日志文件。2）日志文件的命名已经改变，但没有修过日志文件匹配规则。";
            logger.warn(this, title);
            NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogNotExist, title, content);
            event.addArg("serverid", logPatternInfo.getServId());
            event.addArg("appid", logPatternInfo.getAppId());
            logagent.putNotificationEvent(event);
        }
        // <R=filepath,C=logPatternInfo,V=
        tailFileTable.put(logPatternInfo.getAbsolutePath(), logPatternInfo);
        if (logger.isDebugEnable()) {
            logger.debug(this, "update log table. absPath=" + logPatternInfo.getAbsolutePath() + ", info=" + JSONHelper.toString(logPatternInfo));
        }
    }
}