Examples with AlarmRule com.alibaba.otter.shared.common.model.config.alarm.AlarmRule used on opensource projects

Example 1 with AlarmRule

use of com.alibaba.otter.shared.common.model.config.alarm.AlarmRule in project otter by alibaba.

the class DelayStatRuleMonitor method explore.

@Override
public void explore(List<AlarmRule> rules) {
    if (CollectionUtils.isEmpty(rules)) {
        return;
    }
    // 进入到监控项级别的rule，pipelineId一定是相同的
    Long pipelineId = rules.get(0).getPipelineId();
    DelayStat delayStat = delayStatService.findRealtimeDelayStat(pipelineId);
    // seconds
    Long delayTime = 0L;
    Long delayUpdate = 0L;
    if (delayStat.getDelayTime() != null) {
        delayTime = delayStat.getDelayTime() / 1000;
    }
    if (delayStat.getGmtCreate() != null) {
        delayUpdate = (new Date().getTime() - delayStat.getGmtCreate().getTime()) / 1000;
    }
    boolean delayTimeFlag = false;
    boolean delayUpdateFlag = false;
    for (AlarmRule rule : rules) {
        if (rule.getMonitorName().isDelayTime()) {
            delayTimeFlag |= checkDelayTime(rule, delayTime);
            if (delayTimeFlag) {
                //如果出现超时，再check下是否因为最后更新时间过久了
                //检查delay统计的最后更新时间，这也做为delay监控的一部分
                delayUpdateFlag |= checkDelayTime(rule, delayUpdate);
            }
        }
    }
    if (delayTimeFlag && !delayUpdateFlag) {
        logRecordAlarm(pipelineId, MonitorName.DELAYTIME, String.format(DELAY_TIME_MESSAGE, pipelineId, delayTime));
    } else if (delayTimeFlag && delayUpdateFlag) {
        logRecordAlarm(pipelineId, MonitorName.DELAYTIME, String.format(DELAY_TIME_UPDATE_MESSAGE, pipelineId, delayTime, delayUpdate));
    } else if (delayUpdateFlag) {
        logRecordAlarm(pipelineId, MonitorName.DELAYTIME, String.format(DELAY_UPDATE_MESSAGE, pipelineId, delayUpdate));
    }
}

Example 2 with AlarmRule

use of com.alibaba.otter.shared.common.model.config.alarm.AlarmRule in project otter by alibaba.

the class ExceptionRuleMonitor method feed.

@Override
public void feed(Object data, Long pipelineId) {
    if (!(data instanceof NodeAlarmEvent)) {
        return;
    }
    NodeAlarmEvent alarmEvent = (NodeAlarmEvent) data;
    // 异常一定需要记录日志
    String message = String.format(MESAGE_FORMAT, alarmEvent.getPipelineId(), alarmEvent.getNid(), alarmEvent.getMessage());
    logRecordAlarm(pipelineId, alarmEvent.getNid(), MonitorName.EXCEPTION, message);
    // 报警检查
    List<AlarmRule> rules = alarmRuleService.getAlarmRules(pipelineId, AlarmRuleStatus.ENABLE);
    // TODO 需要给 alarmRuleService 提需求
    Date now = new Date();
    List<AlarmRule> exceptionRules = new ArrayList<AlarmRule>();
    for (AlarmRule rule : rules) {
        if (MonitorName.EXCEPTION.equals(rule.getMonitorName()) && checkEnable(rule, now)) {
            exceptionRules.add(rule);
        }
    }
    if (CollectionUtils.isEmpty(exceptionRules)) {
        return;
    }
    for (AlarmRule rule : exceptionRules) {
        check(rule, alarmEvent);
    }
}

Example 3 with AlarmRule

use of com.alibaba.otter.shared.common.model.config.alarm.AlarmRule in project otter by alibaba.

the class PipelineMonitor method explore.

@Override
public void explore(List<AlarmRule> rules) {
    Long pipelineId = rules.get(0).getPipelineId();
    Pipeline pipeline = pipelineService.findById(pipelineId);
    // 如果处于stop状态，则忽略报警
    ChannelStatus status = arbitrateManageService.channelEvent().status(pipeline.getChannelId());
    if (status == null || status.isStop()) {
        return;
    }
    List<AlarmRule> delayTimeRules = new LinkedList<AlarmRule>();
    List<AlarmRule> exceptonRules = new LinkedList<AlarmRule>();
    List<AlarmRule> pipelineTimeoutRules = new LinkedList<AlarmRule>();
    List<AlarmRule> processTimeoutRules = new LinkedList<AlarmRule>();
    List<AlarmRule> positionTimeoutRules = new LinkedList<AlarmRule>();
    Date now = new Date();
    for (AlarmRule rule : rules) {
        switch(rule.getMonitorName()) {
            case DELAYTIME:
                if (checkEnable(rule, now)) {
                    delayTimeRules.add(rule);
                }
                break;
            case EXCEPTION:
                if (checkEnable(rule, now)) {
                    exceptonRules.add(rule);
                }
                break;
            case PIPELINETIMEOUT:
                if (checkEnable(rule, now)) {
                    pipelineTimeoutRules.add(rule);
                }
                break;
            case PROCESSTIMEOUT:
                if (checkEnable(rule, now)) {
                    processTimeoutRules.add(rule);
                }
                break;
            case POSITIONTIMEOUT:
                if (checkEnable(rule, now)) {
                    positionTimeoutRules.add(rule);
                }
                break;
            default:
                break;
        }
    }
    if (!delayTimeRules.isEmpty()) {
        delayStatRuleMonitor.explore(delayTimeRules);
    }
    if (!pipelineTimeoutRules.isEmpty()) {
        pipelineTimeoutRuleMonitor.explore(pipelineTimeoutRules);
    }
    if (!processTimeoutRules.isEmpty()) {
        processTimeoutRuleMonitor.explore(processTimeoutRules);
    }
    if (!positionTimeoutRules.isEmpty()) {
        positionTimeoutRuleMonitor.explore(positionTimeoutRules);
    }
}

Example 4 with AlarmRule

use of com.alibaba.otter.shared.common.model.config.alarm.AlarmRule in project otter by alibaba.

the class PositionTimeoutRuleMonitor method explore.

@Override
public void explore(List<AlarmRule> rules) {
    if (CollectionUtils.isEmpty(rules)) {
        return;
    }
    Long pipelineId = rules.get(0).getPipelineId();
    Pipeline pipeline = pipelineService.findById(pipelineId);
    PositionEventData data = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
    long latestSyncTime = 0L;
    if (data != null && data.getModifiedTime() != null) {
        Date modifiedDate = data.getModifiedTime();
        latestSyncTime = modifiedDate.getTime();
    } else {
        return;
    }
    long now = System.currentTimeMillis();
    long elapsed = now - latestSyncTime;
    boolean flag = false;
    for (AlarmRule rule : rules) {
        flag |= checkTimeout(rule, elapsed);
    }
    if (flag) {
        logRecordAlarm(pipelineId, MonitorName.POSITIONTIMEOUT, String.format(TIME_OUT_MESSAGE, pipelineId, (elapsed / 1000)));
    }
}

Example 5 with AlarmRule

use of com.alibaba.otter.shared.common.model.config.alarm.AlarmRule in project otter by alibaba.

the class ProcessTimeoutRuleMonitor method explore.

@Override
public void explore(List<AlarmRule> rules) {
    if (CollectionUtils.isEmpty(rules)) {
        return;
    }
    Long pipelineId = rules.get(0).getPipelineId();
    List<ProcessStat> processStats = processStatService.listRealtimeProcessStat(pipelineId);
    if (CollectionUtils.isEmpty(processStats)) {
        return;
    }
    long now = System.currentTimeMillis();
    Map<Long, Long> processTime = new HashMap<Long, Long>();
    for (ProcessStat processStat : processStats) {
        Long timeout = 0L;
        if (!CollectionUtils.isEmpty(processStat.getStageStats())) {
            timeout = now - processStat.getStageStats().get(0).getStartTime();
        }
        processTime.put(processStat.getProcessId(), timeout);
    }
    String message = StringUtils.EMPTY;
    for (AlarmRule rule : rules) {
        if (message.isEmpty()) {
            message = checkTimeout(rule, processTime);
        } else {
            checkTimeout(rule, processTime);
        }
    }
    if (!message.isEmpty()) {
        logRecordAlarm(pipelineId, MonitorName.PROCESSTIMEOUT, message);
    }
}