Examples with TaildirLogComponent com.creditease.agent.feature.logagent.TaildirLogComponent used on opensource projects

Example 1 with TaildirLogComponent

use of com.creditease.agent.feature.logagent.TaildirLogComponent in project uavstack by uavorg.

the class LogAgent method notifyProfileDataUpdate.

/**
 * notify there is profile data update
 *
 * @param profileData
 */
@SuppressWarnings("rawtypes")
private void notifyProfileDataUpdate(MonitorDataFrame profileData, JVMAgentInfo agentInfo) {
    /**
     * Step 1: parse MDF to build NewLogPatternInfoMap
     */
    AppLogPatternInfoCollection newLogPatternInfoMap = new AppLogPatternInfoCollection();
    Map<String, LogPatternInfo> newLogPatternPathConflictMap = new HashMap<String, LogPatternInfo>();
    // jvm system properties for ${xxxx} varable
    Properties systemPro = agentInfo.getSystemProperties();
    for (Entry<String, List<Map>> e : profileData.getDatas().entrySet()) {
        // get appid
        String appID = e.getKey();
        // get webapproot
        Map<String, Object> webappInfo = profileData.getElemInstValues(appID, "cpt", "webapp");
        String webAppRoot = (String) webappInfo.get("webapproot");
        if (null == webAppRoot) {
            // notify
            String content = "Application[" + appID + "]'s profileData is incomplete: field[webapproot] is null.";
            log.warn(this, content);
            NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogAppProfileError, "AppProfileError", content);
            event.addArg("serverid", profileData.getServerId());
            event.addArg("appid", appID);
            this.putNotificationEvent(event);
            continue;
        }
        String appurl = (String) webappInfo.get("appurl");
        // get PEId=logs
        List<Map> logsPD = profileData.getElemInstances(appID, "logs");
        for (Map logPD : logsPD) {
            Map logInstanceValues = (Map) logPD.get("values");
            for (Object keyObj : logInstanceValues.keySet()) {
                String logPattern = (String) keyObj;
                // figure out absolute path
                String absPath = figureOutAbsolutePath(webAppRoot, logPattern, systemPro);
                LogPatternInfo lpi = new LogPatternInfo();
                // set appid
                lpi.setAppId(appID);
                // set servid
                lpi.setServId(profileData.getServerId());
                // set logpattern
                lpi.setLogParttern(logPattern);
                // set webapproot
                lpi.setWebAppRoot(webAppRoot);
                // set appurl
                lpi.setAppUrl(appurl);
                // set abspath
                lpi.setAbsolutePath(absPath);
                // set state flag as NEWCOME
                lpi.setFlag(StateFlag.NEWCOME);
                // set timeStamp as current
                lpi.setTimeStamp(System.currentTimeMillis());
                // Step 1.1: check if the logpattern exists in ISSUE MAP
                LogPatternInfo ilpi = this.IssueLogProfileDataMap.get(lpi.getAppUUID(), lpi.getUUID());
                // project may have same serverID,appID,logpattern,but has different absolutepath -- by hongqiangwei
                if (null != ilpi) {
                    // nothing
                    if (ilpi.getAbsolutePath().equalsIgnoreCase(lpi.getAbsolutePath())) {
                        continue;
                    } else // if there is change, then remove it from ISSUE MAP,
                    // this logpattern may go to NEW MAP
                    {
                        this.IssueLogProfileDataMap.remove(lpi.getAppUUID(), lpi.getUUID());
                    }
                }
                // Step 1.2: we should check if there is conflict among the
                // new coming profile data
                boolean isDuplicated = newLogPatternPathConflictMap.containsKey(lpi.getAbsolutePath());
                /**
                 * if duplicated, that means this logpattern has issue and just put it into IssueLogProfileDataMap
                 */
                if (isDuplicated == true) {
                    // move to ISSUE MAP
                    LogPatternInfo templpi = newLogPatternPathConflictMap.get(lpi.getAbsolutePath());
                    LogPatternInfo originlpi = LatestLogProfileDataMap.get(lpi.getAppUUID(), lpi.getUUID());
                    if (originlpi != null) {
                        // remove new(who is not exist in
                        // LatestLogProfileDataMap) from
                        // newLogPatternInfoMap and
                        // newLogPatternPathConflictMap
                        newLogPatternInfoMap.remove(templpi.getAppUUID(), templpi.getUUID());
                        newLogPatternPathConflictMap.remove(templpi.getAbsolutePath());
                        // add new(who is not exist in
                        // LatestLogProfileDataMap) to
                        // IssueLogProfileDataMap
                        this.IssueLogProfileDataMap.put(templpi);
                        // add current(who is exist in
                        // LatestLogProfileDataMap) to newLogPatternInfoMap
                        // and newLogPatternPathConflictMap
                        newLogPatternInfoMap.put(lpi);
                        newLogPatternPathConflictMap.put(lpi.getAbsolutePath(), lpi);
                    } else {
                        this.IssueLogProfileDataMap.put(lpi);
                    }
                    log.err(this, "APPUUID : " + lpi.getAppUUID() + " whoese log absolute path conflict with APPUUID :" + templpi.getAppUUID());
                    String content = "APPUUID : " + lpi.getAppUUID() + " whoese log absolute path conflict with APPUUID :" + templpi.getAppUUID();
                    NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogPathConflict, "LogPathConflict", content);
                    this.putNotificationEvent(event);
                } else /**
                 * if not duplicated, that means this logpattern seems NEWCOME just at this moment.
                 */
                {
                    newLogPatternPathConflictMap.put(lpi.getAbsolutePath(), lpi);
                    // move to NEW MAP
                    newLogPatternInfoMap.put(lpi);
                }
            }
        }
    }
    /**
     * Step 2: Compare newLogPatternInfoMap with LatestLogProfileDataMap to find NEWCOME or UPDATE
     */
    for (Entry<String, Map<String, LogPatternInfo>> entry : newLogPatternInfoMap.entrySet()) {
        String appUUID = entry.getKey();
        Map<String, LogPatternInfo> logPatternMap = entry.getValue();
        boolean lC = this.LatestLogProfileDataMap.containsKey(appUUID);
        for (LogPatternInfo lpi : logPatternMap.values()) {
            // appUUID EXISTs in LatestLogProfileDataMap
            if (lC == true) {
                LogPatternInfo ilpi = this.LatestLogProfileDataMap.get(appUUID, lpi.getUUID());
                // the state should be UPDATE
                if (null != ilpi) {
                    if (!lpi.getAbsolutePath().equals(ilpi.getAbsolutePath())) {
                        lpi.setFlag(StateFlag.UPDATE);
                    } else {
                        lpi.setFlag(StateFlag.EXIST);
                    }
                }
            }
            // add this logpattern to LatestLogProfileDataMap
            // NOTE: currently we are ignoring the abspath's conflicts in
            // LatestLogProfileDataMap,
            // we will handle that in next step
            this.LatestLogProfileDataMap.put(lpi);
        }
    }
    /**
     * Step 3: Compare updated LatestLogProfileDataMap with newLogPatternInfoMap again to find EXIST_UNKOWN
     * logpattern
     */
    for (Entry<String, Map<String, LogPatternInfo>> entry : LatestLogProfileDataMap.entrySet()) {
        String appUUID = entry.getKey();
        Map<String, LogPatternInfo> logPatternMap = entry.getValue();
        boolean isInNew = newLogPatternInfoMap.containsKey(appUUID);
        for (LogPatternInfo lpi : logPatternMap.values()) {
            // appUUID in NEWCOME
            if (isInNew) {
                // check if the logpattern exists in NEWCOME
                LogPatternInfo ilpi = newLogPatternInfoMap.get(appUUID, lpi.getUUID());
                // logUUID not in NEWCOME
                if (ilpi == null) {
                    // mark this logpattern in EXIST_UNKOWN,logCatcher
                    // should fix the state
                    lpi.setFlag(StateFlag.EXIST_UNKOWN);
                }
            } else // appUUID not in NEWCOME
            {
                // mark this logpattern in EXIST_UNKOWN,logCatcher should
                // fix the state
                lpi.setFlag(StateFlag.EXIST_UNKOWN);
            }
        }
    }
    /**
     * Step 4: so sorry, we should rescan the LatestLogProfileDataMap, currently only EXIST_UNKOWN,UPDATE,NEWCOME
     * left to ensure the following conflict rules: 1) UPATE should kick off NEWCOME: 必然存在 2) EXIST_UNKOWN should
     * kick off NEWCOME：尽管可能EXIST_UNKOWN不存在，但保持现状吧，需要logCatcher进行修正, 如果EXIST_UNKOWN存在， 则更新为EXIST，则NEWCOME继续保持在ISSUE
     * MAP 如果EXIST_UNKOWN不存在，则将之移动到ISSUE MAP;其他所有NEWCOME继续保持在ISSUE MAP
     */
    Map<String, List<LogPatternInfo>> logPatternInfoToIssuesMap = new HashMap<String, List<LogPatternInfo>>();
    for (Entry<String, Map<String, LogPatternInfo>> entry : LatestLogProfileDataMap.entrySet()) {
        Map<String, LogPatternInfo> logPatternMap = entry.getValue();
        for (LogPatternInfo lpi : logPatternMap.values()) {
            String absPath = lpi.getAbsolutePath();
            if (!logPatternInfoToIssuesMap.containsKey(absPath)) {
                List<LogPatternInfo> lpiList = new ArrayList<LogPatternInfo>();
                lpiList.add(lpi);
                logPatternInfoToIssuesMap.put(absPath, lpiList);
            } else {
                List<LogPatternInfo> lpiList = logPatternInfoToIssuesMap.get(absPath);
                lpiList.add(lpi);
                /**
                 * sort the list, then every list for one application should have order by flag weight
                 */
                Collections.sort(lpiList, new Comparator<LogPatternInfo>() {

                    @Override
                    public int compare(LogPatternInfo o1, LogPatternInfo o2) {
                        int weightSpan = o2.getFlag().getFlagWeight() - o1.getFlag().getFlagWeight();
                        return weightSpan;
                    }
                });
            }
        }
    }
    /**
     * Step 5: build logCatcherInfoMap
     */
    Map<String, LogPatternInfo> logCatcherInfoMap = new LinkedHashMap<String, LogPatternInfo>();
    for (List<LogPatternInfo> lpiList : logPatternInfoToIssuesMap.values()) {
        LogPatternInfo lpi = lpiList.get(0);
        // we just need the first one as the updated one
        logCatcherInfoMap.put(lpi.getUUID(), lpi);
        // the other ones go to ISSUE MAP
        for (int i = 1; i < lpiList.size(); i++) {
            this.IssueLogProfileDataMap.put(lpiList.get(i));
            LatestLogProfileDataMap.remove(lpiList.get(i).getAppUUID(), lpiList.get(i).getUUID());
        }
    }
    /**
     * in some cases there are may no log pattern infos
     */
    if (logCatcherInfoMap.isEmpty()) {
        log.warn(this, "The logCatcherInfoMap is empty and will not update log pattern info in ProfileData for logCatcher.");
        return;
    }
    /**
     * start logCatcher
     *
     * logCatcher is used to process logs
     */
    if (null == logCatcher) {
        String positionfileroot = this.getConfigManager().getContext(IConfigurationManager.METADATAPATH);
        logCatcher = new TaildirLogComponent("TaildirlogComponent", this.feature);
        // support multiple file in future,add dynamic update tailfiles list
        ConfigurationContext context = new ConfigurationContext();
        context.put(POSITION_FILE_ROOT, positionfileroot);
        try {
            logCatcher.configure(context, logCatcherInfoMap);
            logCatcher.start();
            log.info(this, "ApplicationServer LogCatcher starts SUCCESS");
        } catch (IOException e) {
            log.err(this, "ApplicationServer LogCatcher starts FAIL.", e);
            return;
        }
    } else /**
     * update logCatcher
     */
    {
        logCatcher.configure(logCatcherInfoMap);
        log.info(this, "ApplicationServer LogCatcher updates SUCCESS");
    }
}