Examples with LogAgent com.creditease.agent.feature.LogAgent used on opensource projects

Example 1 with LogAgent

use of com.creditease.agent.feature.LogAgent in project uavstack by uavorg.

the class TaildirLogComponent method tailFileCommon.

@SuppressWarnings("rawtypes")
public void tailFileCommon(TailFile tf, boolean backoffWithoutNL, Map<TailFile, List<Map>> serverlogs) throws IOException, InterruptedException {
    long current = System.currentTimeMillis();
    boolean isEvents = false;
    // while (true) {
    reader.setCurrentFile(tf);
    List<Event> events = reader.readEvents(batchSize, backoffWithoutNL);
    if (!events.isEmpty()) {
        isEvents = true;
    }
    try {
        LogFilterAndRule main = RuleFilterFactory.getInstance().getLogFilterAndRule(tf.getPath());
        List<LogFilterAndRule> aids = RuleFilterFactory.getInstance().getAidLogFilterAndRuleList(tf.getPath());
        List<Map> datalog = RuleFilterFactory.getInstance().createChain(reader, batchSize).setMainLogFilterAndRule(main).setAidLogFilterAndRuleList(aids).doProcess(events, backoffWithoutNL);
        if (!datalog.isEmpty()) {
            if (serverlogs.containsKey(tf)) {
                serverlogs.get(tf).addAll(datalog);
            } else
                serverlogs.put(tf, datalog);
        }
        reader.commit(events.size() < batchSize);
    } catch (IOException ex) {
        log.warn(this, "The unexpected failure. " + "The source will try again after " + retryInterval + " ms");
    // TimeUnit.MILLISECONDS.sleep(retryInterval);
    // retryInterval = retryInterval << 1;
    // retryInter val = Math.min(retryInterval, maxRetryInterval);
    // continue;
    }
    // retryInterval = 1000;
    // if (events.size() < batchSize) {
    // break;
    // }
    // }
    // renew
    LogAgent logagent = (LogAgent) ConfigurationManager.getInstance().getComponent("logagent", "LogAgent");
    LogPatternInfo info = logagent.getLatestLogProfileDataMap().get(tf.getServerId() + "-" + tf.getAppId(), tf.getId());
    if (info != null && isEvents) {
        info.setTimeStamp(current);
        LogPatternInfo innerInfo = reader.getTailFileTable().asMap().get(info.getAbsolutePath());
        if (innerInfo != null) {
            innerInfo.setTimeStamp(current);
            // FIXME concurrent problem
            reader.getTailFileTable().put(innerInfo.getAbsolutePath(), innerInfo);
        }
    }
    if (info != null && current - info.getTimeStamp() > timeOutInterval) {
        logagent.getLatestLogProfileDataMap().remove(tf.getServerId() + "-" + tf.getAppId(), tf.getId());
        // notify
        String title = NetworkHelper.getLocalIP() + "日志[" + tf.getId() + "]的过滤规则配置已经过期.";
        log.err(this, title);
        NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogRuleExpired, title, title);
        event.addArg("serverid", tf.getServerId());
        event.addArg("appid", tf.getAppId());
        this.putNotificationEvent(event);
    }
}

Example 2 with LogAgent

use of com.creditease.agent.feature.LogAgent in project uavstack by uavorg.

the class DoTestTaildirSource method setUp.

@Before
public void setUp() throws IOException {
    SystemLogger.init("DEBUG", true, 5);
    source = new LogAgent("Logagent", "Logagent");
    String os = System.getProperty("os.name");
    if (os.indexOf("Windows") != -1)
        tmpDir = new File("F:/temp");
    else
        tmpDir = new File("/Users/fathead/temp");
    posFilePath = tmpDir.getAbsolutePath() + "/taildir_position_test.json";
    // 1) Create 1st file
    File f1 = new File(tmpDir, "file1");
    // normal sep:\t
    String line1 = "[CE]file1line1chapter1\tfile1line1chapter2\tfile1line1chapter3\n";
    // test [CE]
    String line2 = "[C]file1line2hiddenchapter1\tfile1line2hiddenchapter2\tfile1line2hiddenchapter3\n";
    // filter
    // test
    String line3 = "[CE]file1line3chapter1\tfile1line3chapter2\tfile1line3chapter3\tfile1line3undisplaychapter4\n";
    // plus
    // chapter
    // process
    Files.write(line1 + line2 + line3, f1, Charsets.UTF_8);
    try {
        // wait before creating a new file
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    }
    // 1) Create 2nd file
    // normal sep:|
    String line1b = "<CE>file2line1chapter1|file2line1chapter2|file2line1chapter3\n";
    // test default chapter process
    String line2b = "<CE>file2line2chapter1|file2line2chapter2\n";
    // test plus "" chapter
    String line3b = "<CE>file2line3chapter1|file2line3chapter2\tfile2line3chapter3|\n";
    // process
    File f2 = new File(tmpDir, "file2");
    Files.write(line1b + line2b + line3b, f2, Charsets.UTF_8);
    try {
        // wait before creating next file
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    }
    // 3) Create 3rd file
    // test
    String line1c = "[CE]file3line1hiddenchapter1|file3line1hiddenchapter2|file3line1hiddenchapter3\n";
    // un[CE]
    // filter
    String line2c = "<CE>|file3line2chapter1|\n";
    // test multi- process
    String line3c = "<CE>|file3line3chapter1\n";
    File f3 = new File(tmpDir, "file3");
    Files.write(line1c + line2c + line3c, f3, Charsets.UTF_8);
    // 5) Now update the 3rd file so that its the latest file and gets
    // consumed last
    f3.setLastModified(System.currentTimeMillis());
    // 4) Consume the files
    Map<String, String> logfile = Maps.newHashMap();
    logfile.put("ser1--app1--" + f1.getName(), f1.getPath());
    logfile.put("ser1--app2--" + f2.getName(), f2.getPath());
    logfile.put("ser1--app2--" + f3.getName(), f3.getPath());
    ITimerWorkManager itwm = new SystemTimerWorkMgr();
    ConfigurationManager.getInstance().registerComponent("Global", "ITimerWorkManager", itwm);
    // source.configure(posFilePath, logfile);
    /**
     * constructor filter rule {\"separator\":\"\t\", \"assignfields\":{\"content\":1}, \"timestamp\": 0}
     */
    RuleFilterFactory.getInstance().newBuilder().serverId("ser1").appId("app1").logId(f1.getName()).filterRegex("\\[CE\\].*?").ruleRegex("{\"separator\":\"\t\", \"assignfields\":{\"content1\":1, \"content2\":2,\"content3\":3}, \"timestamp\": 0}").build();
    RuleFilterFactory.getInstance().newBuilder().serverId("ser1").appId("app2").logId(f2.getName()).filterRegex("<CE>.*?").ruleRegex("{\"separator\":\"|\", \"assignfields\":{\"content1\":1, \"content2\":2,\"content3\":3}, \"timestamp\": 0}").build();
    RuleFilterFactory.getInstance().newBuilder().serverId("ser1").appId("app2").logId(f3.getName()).filterRegex("<CE>.*?").ruleRegex("{\"separator\":\"|\", \"assignfields\":{\"content1\":1, \"content2\":2,\"content3\":3}, \"timestamp\": 0}").build();
    source.start();
}

Example 3 with LogAgent

use of com.creditease.agent.feature.LogAgent in project uavstack by uavorg.

the class ReliableTaildirEventReader method updateTailFiles.

/**
 * Update tailFiles mapping if a new file is created or appends are detected to the existing file.
 */
public List<Long> updateTailFiles(boolean skipToEnd) throws IOException {
    LogAgent logagent = (LogAgent) ConfigurationManager.getInstance().getComponent("logagent", "LogAgent");
    updateTime = System.currentTimeMillis();
    List<Long> updatedInodes = Lists.newArrayList();
    String serverid = null;
    String appid = null;
    String logid = null;
    String appurl = null;
    for (Entry<String, LogPatternInfo> cell : tailFileTable.asMap().entrySet()) {
        // cell<serverid--appid--logid, logpath, logname>
        // 
        Map<String, String> headers = headerTable.row(cell.getKey());
        LogPatternInfo logPatternInfo = cell.getValue();
        // 文件父路径
        File parentDir = logPatternInfo.getParentDir();
        // 编译后的文件名
        Pattern fileNamePattern = logPatternInfo.getLogRegxPattern();
        serverid = logPatternInfo.getServId();
        appid = logPatternInfo.getAppId();
        logid = logPatternInfo.getLogParttern();
        appurl = logPatternInfo.getAppUrl();
        List<File> files = getMatchFiles(parentDir, fileNamePattern);
        LogPatternInfo logPatternInfo2 = logagent.getLatestLogProfileDataMap().get(logPatternInfo.getAppUUID(), logPatternInfo.getUUID());
        if (!files.isEmpty()) {
            // modify status UNKNOWN to EXISTS
            if (logPatternInfo2 != null) {
                logPatternInfo2.setFlag(StateFlag.EXIST);
            }
        } else if (logPatternInfo2.getFlag() == StateFlag.EXIST) {
            logPatternInfo2.setFlag(StateFlag.EXIST_UNKOWN);
            String title = NetworkHelper.getLocalIP() + "曾经在" + logPatternInfo.getParentDir() + "符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]的日志文件消失了。";
            String content = "失败原因：1）错误删除了这些日志文件。2）修改了日志文件名称，且新名称不符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]。";
            logger.warn(this, title);
            NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogNotExist, title, content);
            event.addArg("serverid", logPatternInfo.getServId());
            event.addArg("appid", logPatternInfo.getAppId());
            logagent.putNotificationEvent(event);
        }
        for (File f : files) {
            long inode = getInode(f);
            TailFile tf = tailFiles.get(inode);
            if (tf == null || !tf.getPath().equals(f.getAbsolutePath())) {
                // 第一次读取从头开始读
                long startPos = skipToEnd ? f.length() : 0;
                // how to get line's number ?
                long startNum = 0;
                // try to get pos form position file
                if (maybeReloadMap.containsKey(inode)) {
                    startPos = maybeReloadMap.get(inode)[0];
                    startNum = maybeReloadMap.get(inode)[1];
                }
                tf = openFile(serverid, appid, logid, f, headers, inode, startPos, startNum);
                tf.setAppUrl(appurl);
            } else {
                boolean updated = tf.getLastUpdated() < f.lastModified();
                if (updated) {
                    if (tf.getRaf() == null) {
                        // 获取文件的读取手柄
                        tf = openFile(serverid, appid, logid, f, headers, inode, tf.getPos(), tf.getNum());
                        tf.setAppUrl(appurl);
                    }
                    if (f.length() < tf.getPos()) {
                        // 文件的长度小于上次读取的指针说明文件内容被删除了，改成从0读取
                        logger.info(this, "Pos " + tf.getPos() + " is larger than file size! " + "Restarting from pos 0, file: " + tf.getPath() + ", inode: " + inode);
                        tf.updatePos(tf.getPath(), inode, 0, 0);
                    }
                }
                // 设置是否需要监控指标
                tf.setNeedTail(updated);
            }
            tailFiles.put(inode, tf);
            updatedInodes.add(inode);
            if (logger.isDebugEnable()) {
                logger.debug(this, "tailfile mapping: " + inode + " --> " + tf.getId());
            }
        }
    }
    return updatedInodes;
}

Example 4 with LogAgent

use of com.creditease.agent.feature.LogAgent in project uavstack by uavorg.

the class ReliableTaildirEventReader method updatelog.

public void updatelog(Map<String, LogPatternInfo> filePaths) {
    for (Entry<String, LogPatternInfo> e : filePaths.entrySet()) {
        LogPatternInfo logPatternInfo = e.getValue();
        LogAgent logagent = (LogAgent) ConfigurationManager.getInstance().getComponent("logagent", "LogAgent");
        LogPatternInfo logPatternInfoTemp = logagent.getLatestLogProfileDataMap().get(logPatternInfo.getAppUUID(), logPatternInfo.getUUID());
        List<File> list = getMatchFiles(logPatternInfo.getParentDir(), logPatternInfo.getLogRegxPattern());
        if (!list.isEmpty()) {
            logPatternInfoTemp.setFlag(StateFlag.EXIST);
        } else {
            logPatternInfoTemp.setFlag(StateFlag.EXIST_UNKOWN);
        }
        if (list.isEmpty() && !logPatternInfo.getParentDir().isDirectory()) {
            // notify
            String title = NetworkHelper.getLocalIP() + "在" + logPatternInfo.getParentDir() + "下没有符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]的日志文件。";
            String content = "失败原因：1）日志文件匹配规则配置错误，所以不能定位日志文件。2）日志文件的命名已经改变，但没有修过日志文件匹配规则。";
            logger.warn(this, title);
            NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogNotExist, title, content);
            event.addArg("serverid", logPatternInfo.getServId());
            event.addArg("appid", logPatternInfo.getAppId());
            logagent.putNotificationEvent(event);
        }
        // <R=filepath,C=logPatternInfo,V=
        tailFileTable.put(logPatternInfo.getAbsolutePath(), logPatternInfo);
        if (logger.isDebugEnable()) {
            logger.debug(this, "update log table. absPath=" + logPatternInfo.getAbsolutePath() + ", info=" + JSONHelper.toString(logPatternInfo));
        }
    }
}