Examples with NotificationEvent com.creditease.agent.monitor.api.NotificationEvent used on opensource projects

Example 11 with NotificationEvent

use of com.creditease.agent.monitor.api.NotificationEvent in project uavstack by uavorg.

the class JudgeNotifyTask method run.

@Override
public void run() {
    NotifyStrategy stra = null;
    try {
        /**
         * Step 1: seek strategy
         */
        RuntimeNotifyStrategyMgr strategyMgr = (RuntimeNotifyStrategyMgr) getConfigManager().getComponent(this.feature, "RuntimeNotifyStrategyMgr");
        stra = strategyMgr.seekStrategy(curSlice.getKey());
        if (stra == null) {
            return;
        }
        /**
         * Step 1.5: underclocking
         */
        long range = stra.getMaxRange();
        if (range > 0) {
            CacheManager cm = (CacheManager) getConfigManager().getComponent(this.feature, STORAGE_CACHE_MANAGER_NAME);
            String judgedKey = genJudgedKey(curSlice);
            if (cm.exists(UAV_CACHE_REGION, judgedKey)) {
                return;
            } else {
                cm.put(UAV_CACHE_REGION, judgedKey, String.valueOf(curSlice.getTime()));
                cm.expire(UAV_CACHE_REGION, judgedKey, range, TimeUnit.MILLISECONDS);
            }
        }
        /**
         * Step 2: dump range slices
         */
        List<Slice> rangeSlices = null;
        if (range > 0) {
            RuntimeNotifySliceMgr sliceMgr = (RuntimeNotifySliceMgr) getConfigManager().getComponent(this.feature, "RuntimeNotifySliceMgr");
            rangeSlices = sliceMgr.getSlices(curSlice, range);
            if (rangeSlices.isEmpty()) {
                if (log.isDebugEnable()) {
                    log.debug(this, "RuntimeNotify judge dump invalid.");
                }
                return;
            }
        } else {
            rangeSlices = new ArrayList<>(1);
            rangeSlices.add(curSlice);
        }
        /**
         * Step 3: judge the strategy
         */
        StrategyJudgement judgement = (StrategyJudgement) getConfigManager().getComponent(feature, "StrategyJudgement");
        Map<String, String> result = judgement.judge(curSlice, stra, rangeSlices);
        // ?? maybe no effective
        if (rangeSlices != null) {
            rangeSlices.clear();
            rangeSlices = null;
        }
        /**
         * Step 5: if fire the event, build notification event
         */
        if (result != null && !result.isEmpty()) {
            NotificationEvent event = this.newNotificationEvent(result, stra.getConvergences());
            // get context
            putContext(event);
            // get action
            putNotifyAction(event, stra);
            // get msg tempalte
            putNotifyMsg(event, stra);
            if (this.log.isTraceEnable()) {
                this.log.info(this, "RuntimeNotify Notification Event Happen: event=" + event.toJSONString());
            }
            this.putNotificationEvent(event);
        }
    } catch (Exception e) {
        log.err(this, "JudgeNotifyTask RUN FAIL.", e);
    }
    if (log.isDebugEnable()) {
        long cost = System.currentTimeMillis() - taskStart;
        String detail = cost < 10 ? "" : " detail: key=" + curSlice.getKey() + ", strategy=" + JSONHelper.toString(stra);
        log.debug(this, "whole task lifecycle COST: (" + cost + ")ms" + detail);
    }
}

Example 12 with NotificationEvent

use of com.creditease.agent.monitor.api.NotificationEvent in project uavstack by uavorg.

the class JudgeNotifyTimerTask method run.

@Override
public void run() {
    CacheLock lock = null;
    try {
        lock = cm.newCacheLock(LOCK_REGION, stra.getName(), LOCK_TIMEOUT);
        if (!lock.getLock()) {
            return;
        }
        /**
         * Step 1:find out instance
         */
        for (String instance : stra.getInstances()) {
            /**
             * Step 2: judge the strategy
             */
            StrategyJudgement judgement = (StrategyJudgement) getConfigManager().getComponent(feature, "StrategyJudgement");
            Map<String, String> result = judgement.judge(new Slice(instance, judge_time), stra, null);
            /**
             * Step 3: if fire the event, build notification event
             */
            if (result != null && !result.isEmpty()) {
                NotificationEvent event = this.newNotificationEvent(instance, result, stra.getConvergences());
                // get context
                putContext(event);
                // get action
                putNotifyAction(event, stra);
                // get msg tempalte
                putNotifyMsg(event, stra);
                if (this.log.isTraceEnable()) {
                    this.log.info(this, "RuntimeNotify Notification Event Happen: event=" + event.toJSONString());
                }
                this.putNotificationEvent(event);
            }
        }
    } catch (Exception e) {
        log.err(this, "JudgeNotifyTimerTask" + stra.getName() + " RUN FAIL.", e);
    } finally {
        if (lock != null && lock.isLockInHand()) {
            lock.releaseLock();
        }
    }
    if (log.isDebugEnable()) {
        long cost = System.currentTimeMillis() - taskStart;
        String detail = cost < 10 ? "" : " detail:strategy=" + JSONHelper.toString(stra);
        log.debug(this, "whole task lifecycle COST: (" + cost + ")ms" + detail);
    }
}

Example 13 with NotificationEvent

use of com.creditease.agent.monitor.api.NotificationEvent in project uavstack by uavorg.

the class NodeInfoWatcher method fireEvent.

/**
 * 触发预警事件
 */
private void fireEvent(Map<String, Map<String, String>> deadProcs) {
    /**
     * Step 1: split crash event by IP
     */
    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    Map<String, CrashEventObj> ips = new HashMap<String, CrashEventObj>();
    for (Entry<String, Map<String, String>> en : deadProcs.entrySet()) {
        String procKey = en.getKey();
        String[] procInfo = procKey.split("_", -1);
        String ip = procInfo[0];
        String procName = procInfo[1];
        String deadtime = en.getValue().get("deadtime");
        String appgroup = en.getValue().get("appgroup");
        CrashEventObj ceo;
        if (!ips.containsKey(ip)) {
            ceo = new CrashEventObj(ip, appgroup);
            ips.put(ip, ceo);
        } else {
            ceo = ips.get(ip);
        }
        ceo.increDeadProcsCount();
        ceo.addDeadProcName(procName);
        ceo.addDeadProcInfo("触发时间：" + format.format(new Date(Long.parseLong(deadtime))) + ", 进程信息：" + procKey);
    }
    /**
     * Step 2: send notification event by IP
     */
    RuntimeNotifyStrategyMgr strategyMgr = (RuntimeNotifyStrategyMgr) getConfigManager().getComponent(this.feature, "RuntimeNotifyStrategyMgr");
    for (CrashEventObj ceo : ips.values()) {
        String title = "应用组[" + ceo.getAppGroup() + "]的" + ceo.getIp() + "共发现" + ceo.getDeadProcsCount() + "进程" + ceo.getDeadProcNamesAsString() + "可疑死掉";
        String description = ceo.getDeadProcsInfoAsString();
        NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_RT_ALERT_CRASH, title, description, System.currentTimeMillis(), ceo.getIp(), "");
        /**
         * Notification Manager will not block the event, the frozen time has no effect to this event
         */
        event.addArg(NotificationEvent.EVENT_Tag_NoBlock, "true");
        // add appgroup
        event.addArg("appgroup", ceo.getAppGroup());
        NotifyStrategy stra = strategyMgr.seekStrategy("server@procCrash@" + ceo.getIp());
        if (null != stra) {
            putNotifyAction(event, stra);
        }
        if (log.isTraceEnable()) {
            log.info(this, "NodeInfoWatcher Crash Event Happen: event=" + event.toJSONString());
        }
        this.putNotificationEvent(event);
    }
}

Example 14 with NotificationEvent

use of com.creditease.agent.monitor.api.NotificationEvent in project uavstack by uavorg.

the class JudgeNotifyTask method newNotificationEvent.

/**
 * newNotificationEvent
 *
 * @return
 */
private NotificationEvent newNotificationEvent(Map<String, String> result, List<String> convergences) {
    String ip = this.curSlice.getMdf().getIP();
    String host = this.curSlice.getMdf().getHost();
    String appgroup = this.curSlice.getMdf().getExt("appgroup");
    appgroup = (appgroup == null) ? "" : appgroup;
    StringBuilder desc = new StringBuilder();
    List<String> conditionIndex = new ArrayList<String>();
    for (Map.Entry<String, String> cause : result.entrySet()) {
        // description
        desc.append("触发条件[" + cause.getKey() + "]：").append(cause.getValue()).append("\r\n");
        // condition index
        conditionIndex.add(cause.getKey());
    }
    String title = ip + "[" + this.curSlice.getKey() + "]触发" + result.size() + "个报警(条件序号： " + conditionIndex.toString().replaceAll("\\[|]|,", "") + ")";
    // fix &nbsp(\u00A0) can be shown in email
    String description = desc.toString().replace('\u00A0', ' ');
    NotificationEvent ne = new NotificationEvent(NotificationEvent.EVENT_RT_ALERT_THRESHOLD, title, description, curSlice.getTime(), ip, host);
    // add appgroup
    ne.addArg("appgroup", appgroup);
    // 兼容不存在convergences属性的旧预警策略
    if (convergences == null || convergences.size() == 0) {
        return ne;
    }
    // 同一个Event由多个策略触发时，梯度收敛以最长的为准
    String conv = obtainConvergenceForEvent(convergences, conditionIndex);
    if (!StringHelper.isEmpty(conv)) {
        ne.addArg("convergences", conv);
        ne.addArg(NotificationEvent.EVENT_Tag_NoBlock, "true");
    }
    return ne;
}

Example 15 with NotificationEvent

use of com.creditease.agent.monitor.api.NotificationEvent in project uavstack by uavorg.

the class ReliableTaildirEventReader method updateTailFiles.

/**
 * Update tailFiles mapping if a new file is created or appends are detected to the existing file.
 */
public List<Long> updateTailFiles(boolean skipToEnd) throws IOException {
    LogAgent logagent = (LogAgent) ConfigurationManager.getInstance().getComponent("logagent", "LogAgent");
    updateTime = System.currentTimeMillis();
    List<Long> updatedInodes = Lists.newArrayList();
    String serverid = null;
    String appid = null;
    String logid = null;
    String appurl = null;
    for (Entry<String, LogPatternInfo> cell : tailFileTable.asMap().entrySet()) {
        // cell<serverid--appid--logid, logpath, logname>
        // 
        Map<String, String> headers = headerTable.row(cell.getKey());
        LogPatternInfo logPatternInfo = cell.getValue();
        // 文件父路径
        File parentDir = logPatternInfo.getParentDir();
        // 编译后的文件名
        Pattern fileNamePattern = logPatternInfo.getLogRegxPattern();
        serverid = logPatternInfo.getServId();
        appid = logPatternInfo.getAppId();
        logid = logPatternInfo.getLogParttern();
        appurl = logPatternInfo.getAppUrl();
        List<File> files = getMatchFiles(parentDir, fileNamePattern);
        LogPatternInfo logPatternInfo2 = logagent.getLatestLogProfileDataMap().get(logPatternInfo.getAppUUID(), logPatternInfo.getUUID());
        if (!files.isEmpty()) {
            // modify status UNKNOWN to EXISTS
            if (logPatternInfo2 != null) {
                logPatternInfo2.setFlag(StateFlag.EXIST);
            }
        } else if (logPatternInfo2.getFlag() == StateFlag.EXIST) {
            logPatternInfo2.setFlag(StateFlag.EXIST_UNKOWN);
            String title = NetworkHelper.getLocalIP() + "曾经在" + logPatternInfo.getParentDir() + "符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]的日志文件消失了。";
            String content = "失败原因：1）错误删除了这些日志文件。2）修改了日志文件名称，且新名称不符合日志文件匹配规则[" + logPatternInfo.getLogRegxPattern() + "]。";
            logger.warn(this, title);
            NotificationEvent event = new NotificationEvent(NotificationEvent.EVENT_LogNotExist, title, content);
            event.addArg("serverid", logPatternInfo.getServId());
            event.addArg("appid", logPatternInfo.getAppId());
            logagent.putNotificationEvent(event);
        }
        for (File f : files) {
            long inode = getInode(f);
            TailFile tf = tailFiles.get(inode);
            if (tf == null || !tf.getPath().equals(f.getAbsolutePath())) {
                // 第一次读取从头开始读
                long startPos = skipToEnd ? f.length() : 0;
                // how to get line's number ?
                long startNum = 0;
                // try to get pos form position file
                if (maybeReloadMap.containsKey(inode)) {
                    startPos = maybeReloadMap.get(inode)[0];
                    startNum = maybeReloadMap.get(inode)[1];
                }
                tf = openFile(serverid, appid, logid, f, headers, inode, startPos, startNum);
                tf.setAppUrl(appurl);
            } else {
                boolean updated = tf.getLastUpdated() < f.lastModified();
                if (updated) {
                    if (tf.getRaf() == null) {
                        // 获取文件的读取手柄
                        tf = openFile(serverid, appid, logid, f, headers, inode, tf.getPos(), tf.getNum());
                        tf.setAppUrl(appurl);
                    }
                    if (f.length() < tf.getPos()) {
                        // 文件的长度小于上次读取的指针说明文件内容被删除了，改成从0读取
                        logger.info(this, "Pos " + tf.getPos() + " is larger than file size! " + "Restarting from pos 0, file: " + tf.getPath() + ", inode: " + inode);
                        tf.updatePos(tf.getPath(), inode, 0, 0);
                    }
                }
                // 设置是否需要监控指标
                tf.setNeedTail(updated);
            }
            tailFiles.put(inode, tf);
            updatedInodes.add(inode);
            if (logger.isDebugEnable()) {
                logger.debug(this, "tailfile mapping: " + inode + " --> " + tf.getId());
            }
        }
    }
    return updatedInodes;
}