Examples with LogFilterAndRule com.creditease.agent.feature.logagent.api.LogFilterAndRule used on opensource projects

Example 1 with LogFilterAndRule

use of com.creditease.agent.feature.logagent.api.LogFilterAndRule in project uavstack by uavorg.

the class TaildirLogComponent method tailFileCommon.

@SuppressWarnings("rawtypes")
public void tailFileCommon(TailFile tf, boolean backoffWithoutNL, Map<TailFile, List<Map>> serverlogs) throws IOException, InterruptedException {
    long current = System.currentTimeMillis();
    boolean isEvents = false;
    // while (true) {
    reader.setCurrentFile(tf);
    List<Event> events = reader.readEvents(batchSize, backoffWithoutNL);
    if (!events.isEmpty()) {
        isEvents = true;
    }
    try {
        LogFilterAndRule main = RuleFilterFactory.getInstance().getLogFilterAndRule(tf.getPath());
        List<LogFilterAndRule> aids = RuleFilterFactory.getInstance().getAidLogFilterAndRuleList(tf.getPath());
        List<Map> datalog = RuleFilterFactory.getInstance().createChain(reader, batchSize).setMainLogFilterAndRule(main).setAidLogFilterAndRuleList(aids).doProcess(events, backoffWithoutNL);
        if (!datalog.isEmpty()) {
            if (serverlogs.containsKey(tf)) {
                serverlogs.get(tf).addAll(datalog);
            } else
                serverlogs.put(tf, datalog);
        }
        reader.commit(events.size() < batchSize);
    } catch (IOException ex) {
        log.warn(this, "The unexpected failure. " + "The source will try again after " + retryInterval + " ms");
    // TimeUnit.MILLISECONDS.sleep(retryInterval);
    // retryInterval = retryInterval << 1;
    // retryInter val = Math.min(retryInterval, maxRetryInterval);
    // continue;
    }
    // retryInterval = 1000;
    // if (events.size() < batchSize) {
    // break;
    // }
    // }
    // renew
    LogAgent logagent = (LogAgent) ConfigurationManager.getInstance().getComponent("logagent", "LogAgent");
    LogPatternInfo info = logagent.getLatestLogProfileDataMap().get(tf.getServerId() + "-" + tf.getAppId(), tf.getId());
    if (info != null && isEvents) {
        info.setTimeStamp(current);
        LogPatternInfo innerInfo = reader.getTailFileTable().asMap().get(info.getAbsolutePath());
        if (innerInfo != null) {
            innerInfo.setTimeStamp(current);
            // FIXME concurrent problem
            reader.getTailFileTable().put(innerInfo.getAbsolutePath(), innerInfo);
        }
    }
    if (info != null && current - info.getTimeStamp() > timeOutInterval) {
        logagent.getLatestLogProfileDataMap().remove(tf.getServerId() + "-" + tf.getAppId(), tf.getId());
        // notify
        String title = NetworkHelper.getLocalIP() + "日志[" + tf.getId() + "]的过滤规则配置已经过期.";
        log.err(this, title);
        NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogRuleExpired, title, title);
        event.addArg("serverid", tf.getServerId());
        event.addArg("appid", tf.getAppId());
        this.putNotificationEvent(event);
    }
}

Example 2 with LogFilterAndRule

use of com.creditease.agent.feature.logagent.api.LogFilterAndRule in project uavstack by uavorg.

the class DoTestRuleFilterFactory method getDefaultInstance.

@Test
public void getDefaultInstance() {
    String classname = "Default";
    String rule = null;
    rule = Optional.fromNullable(rule).or("{\"separator\":\"\t\", \"assignfields\":{\"content\":1}, \"timestamp\": 0}");
    // parse filter json
    String filter = null;
    String filterregex = Optional.fromNullable(filter).or(".*");
    // parse rule json
    JSONObject robject = JSON.parseObject(rule);
    String separator = Optional.fromNullable(robject.getString("separator")).or("\t");
    JSONObject assignFields = Optional.fromNullable(robject.getJSONObject("assignfields")).or(JSON.parseObject("{content:1}"));
    // Verify timeStamp number is available
    int timestampNumber = robject.getIntValue("timestamp");
    LogFilterAndRule mainLogFAR = (LogFilterAndRule) ReflectionHelper.newInstance("com.creditease.agent.feature.logagent.far." + classname + "LogFilterAndRule", new Class[] { String.class, String.class, JSONObject.class, int.class }, new Object[] { filterregex, separator, assignFields, timestampNumber });
    assertNotNull(mainLogFAR);
    System.out.println(mainLogFAR);
}

Example 3 with LogFilterAndRule

use of com.creditease.agent.feature.logagent.api.LogFilterAndRule in project uavstack by uavorg.

the class LogAgent method updateAllStrategy.

@SuppressWarnings({ "rawtypes", "unchecked" })
private void updateAllStrategy(String stragetyJson) {
    // full stragety
    List<Map> list = JSONHelper.toObjectArray(stragetyJson, Map.class);
    if (list == null || list.size() == 0) {
        log.warn(this, "stragety json is empty!");
        return;
    }
    for (Map m : list) {
        String serverid = (String) m.get("servid");
        String appid = (String) m.get("appid");
        String logid = (String) m.get("logid");
        String filter = (String) m.get("filter");
        String separator = (String) m.get("separator");
        String fields = (String) m.get("fields");
        LogPatternInfo lcfg = new LogPatternInfo();
        lcfg.setServId(serverid);
        lcfg.setAppId(appid);
        lcfg.setLogParttern(logid);
        LogPatternInfo info = this.LatestLogProfileDataMap.get(lcfg.getAppUUID(), lcfg.getUUID());
        if (info == null) {
            log.warn(this, "log pattern info is null! loguuid:" + lcfg.getUUID());
            return;
        }
        String logPath = info.getAbsolutePath();
        Map mapping = new HashMap();
        mapping.putAll(m);
        mapping.put("absPath", logPath);
        logCfgMapping.put(lcfg.getUUID(), mapping);
        LogFilterAndRule lfar = new DefaultLogFilterAndRule(filter, separator, JSON.parseObject(fields), 0, 0);
        RuleFilterFactory.getInstance().pubLogFilterAndRule(logPath, lfar);
    }
    saveLogCfg();
}

Example 4 with LogFilterAndRule

use of com.creditease.agent.feature.logagent.api.LogFilterAndRule in project uavstack by uavorg.

the class LogAgent method loadLogCfg.

private void loadLogCfg() {
    String cfgFile = getLogCfgFile();
    String cfg = IOHelper.readTxtFile(cfgFile, "utf-8");
    if (StringHelper.isEmpty(cfg)) {
        return;
    }
    @SuppressWarnings({ "unchecked", "rawtypes" }) Map<String, Map> mapping = JSONHelper.toObject(cfg, Map.class);
    logCfgMapping.putAll(mapping);
    for (@SuppressWarnings("rawtypes") Map m : logCfgMapping.values()) {
        String absPath = (String) m.get("absPath");
        String filter = (String) m.get("filter");
        String separator = (String) m.get("separator");
        String fields = (String) m.get("fields");
        LogFilterAndRule lfar = new DefaultLogFilterAndRule(filter, separator, JSON.parseObject(fields), 0, 0);
        RuleFilterFactory.getInstance().pubLogFilterAndRule(absPath, lfar);
    }
}

Example 5 with LogFilterAndRule

use of com.creditease.agent.feature.logagent.api.LogFilterAndRule in project uavstack by uavorg.

the class RuleFilterFactory method getLogFilterAndRule.

// this key should be absFilePath --- by hongqiang
public LogFilterAndRule getLogFilterAndRule(String id) {
    // id = id.replace('\\', '/');
    LogFilterAndRule lfar = null;
    lfar = logcollection.getIfPresent(id);
    if (lfar != null) {
        return lfar;
    }
    try {
        lfar = logcollection.get(id, new Callable<LogFilterAndRule>() {

            @Override
            public LogFilterAndRule call() {
                return DEFAULT_LOGFAR;
            }
        });
    } catch (ExecutionException e) {
    }
    return lfar;
}