Examples with PositionEventData com.alibaba.otter.shared.arbitrate.model.PositionEventData used on opensource projects

Example 1 with PositionEventData

use of com.alibaba.otter.shared.arbitrate.model.PositionEventData in project otter by alibaba.

the class PositionTimeoutRuleMonitor method explore.

@Override
public void explore(List<AlarmRule> rules) {
    if (CollectionUtils.isEmpty(rules)) {
        return;
    }
    Long pipelineId = rules.get(0).getPipelineId();
    Pipeline pipeline = pipelineService.findById(pipelineId);
    PositionEventData data = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
    long latestSyncTime = 0L;
    if (data != null && data.getModifiedTime() != null) {
        Date modifiedDate = data.getModifiedTime();
        latestSyncTime = modifiedDate.getTime();
    } else {
        return;
    }
    long now = System.currentTimeMillis();
    long elapsed = now - latestSyncTime;
    boolean flag = false;
    for (AlarmRule rule : rules) {
        flag |= checkTimeout(rule, elapsed);
    }
    if (flag) {
        logRecordAlarm(pipelineId, MonitorName.POSITIONTIMEOUT, String.format(TIME_OUT_MESSAGE, pipelineId, (elapsed / 1000)));
    }
}

Example 2 with PositionEventData

use of com.alibaba.otter.shared.arbitrate.model.PositionEventData in project otter by alibaba.

the class PositionAction method doRemove.

public void doRemove(@Param("pipelineId") Long pipelineId, Navigator nav) throws Exception {
    Pipeline pipeline = pipelineService.findById(pipelineId);
    String destination = pipeline.getParameters().getDestinationName();
    short clientId = pipeline.getParameters().getMainstemClientId();
    PositionEventData position = arbitrateViewService.getCanalCursor(destination, clientId);
    // 记录一下日志
    logger.warn("remove pipelineId[{}] position \n {}", pipelineId, position);
    arbitrateViewService.removeCanalCursor(destination, clientId);
    nav.redirectToLocation("analysisStageStat.htm?pipelineId=" + pipelineId);
}

Example 3 with PositionEventData

use of com.alibaba.otter.shared.arbitrate.model.PositionEventData in project otter by alibaba.

the class PipelineList method execute.

public void execute(@Param("channelId") Long channelId, @Param("pipelineId") Long pipelineId, HttpSession session, Context context) throws Exception {
    Channel channel = channelService.findByIdWithoutColumn(channelId);
    List<Pipeline> pipelines = channel.getPipelines();
    List<Pipeline> tempPipe = new ArrayList<Pipeline>();
    if ((pipelineId != null) && (pipelineId != 0l)) {
        for (Pipeline pipeline : pipelines) {
            if (!pipeline.getId().equals(pipelineId)) {
                tempPipe.add(pipeline);
            }
        }
        pipelines.removeAll(tempPipe);
    }
    Map<Long, DelayStat> delayStats = new HashMap<Long, DelayStat>(pipelines.size(), 1f);
    Map<Long, MainStemEventData> mainstemDatas = new HashMap<Long, MainStemEventData>(pipelines.size(), 1f);
    Map<Long, ThroughputStat> throughputStats = new HashMap<Long, ThroughputStat>(pipelines.size(), 1f);
    Map<Long, List<AlarmRule>> alarmRuleStats = new HashMap<Long, List<AlarmRule>>(pipelines.size(), 1f);
    Map<Long, PositionEventData> positionDatas = new HashMap<Long, PositionEventData>(pipelines.size(), 1f);
    for (Pipeline pipeline : pipelines) {
        DelayStat delayStat = delayStatService.findRealtimeDelayStat(pipeline.getId());
        if (delayStat.getDelayNumber() == null) {
            delayStat.setDelayNumber(0L);
            delayStat.setDelayTime(0L);
            delayStat.setGmtModified(pipeline.getGmtModified());
        }
        delayStats.put(pipeline.getId(), delayStat);
        mainstemDatas.put(pipeline.getId(), arbitrateViewService.mainstemData(channel.getId(), pipeline.getId()));
        ThroughputCondition condition = new ThroughputCondition();
        condition.setPipelineId(pipeline.getId());
        condition.setType(ThroughputType.ROW);
        ThroughputStat throughputStat = throughputStatService.findThroughputStatByPipelineId(condition);
        throughputStats.put(pipeline.getId(), throughputStat);
        List<AlarmRule> alarmRules = alarmRuleService.getAlarmRules(pipeline.getId());
        alarmRuleStats.put(pipeline.getId(), alarmRules);
        PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
        positionDatas.put(pipeline.getId(), positionData);
    }
    context.put("channel", channel);
    context.put("pipelines", pipelines);
    context.put("delayStats", delayStats);
    context.put("throughputStats", throughputStats);
    context.put("alarmRuleStats", alarmRuleStats);
    context.put("mainstemDatas", mainstemDatas);
    context.put("positionDatas", positionDatas);
}

Example 4 with PositionEventData

use of com.alibaba.otter.shared.arbitrate.model.PositionEventData in project otter by alibaba.

the class AnalysisStageStat method execute.

public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {
    List<ProcessStat> processStats = new ArrayList<ProcessStat>();
    Pipeline pipeline = pipelineService.findById(pipelineId);
    processStats = processStatService.listRealtimeProcessStat(pipelineId);
    // Map ext = new HashMap<Long, String>();
    // // ext.put(145456451, "asdf");
    // for (Long i = 1L; i <= 3; i++) {
    // List<StageStat> stageStats = new ArrayList<StageStat>();
    // ProcessStat processStat = new ProcessStat();
    // processStat.setPipelineId(1L);
    // processStat.setProcessId(i);
    // StageStat stage = new StageStat();
    // stage.setStage(StageType.SELECT);
    // stage.setStartTime(((new Date()).getTime() + i * 10 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 200 * 1000));
    // stage.setNumber(11231230L);
    // stage.setSize(14545645640L);
    // // stage.setExts(ext);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.EXTRACT);
    // stage.setStartTime(((new Date()).getTime() + i * 2000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 3000 * 1000));
    // stage.setExts(ext);
    // // stage.setNumber(10L);
    // // stage.setSize(10L);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.TRANSFORM);
    // stage.setStartTime(((new Date()).getTime() + i * 5000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 6000 * 1000));
    // stage.setNumber(154640L);
    // stage.setExts(ext);
    // // stage.setSize(10L);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.LOAD);
    // stage.setStartTime(((new Date()).getTime() + i * 70000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 80000 * 1000));
    // // stage.setNumber(10L);
    // stage.setSize(101445L);
    // // stage.setExts(ext);
    // stageStats.add(stage);
    // processStat.setStageStats(stageStats);
    // processStats.add(processStat);
    // }
    Long stageStart = 0L;
    // Long stageEnd = new Date().getTime() + 3 * 80000 * 1000;
    Long stageEnd = new Date().getTime();
    Long interval = 0L;
    double offset = 0L;
    // 找出最先开始的process的select阶段的开始时间作为起始时间
    if (processStats.size() > 0) {
        if (processStats.get(0).getStageStats().size() > 0) {
            stageStart = processStats.get(0).getStageStats().get(0).getStartTime();
        }
    }
    // 动态计算每个阶段的长度比例
    if (stageStart > 0) {
        interval = stageEnd - stageStart;
    }
    if (interval > 0) {
        offset = 800.0 / interval;
    }
    // 计算每个process当前任务所做的时间总和
    Map<Long, Long> processTime = new HashMap<Long, Long>();
    for (ProcessStat processStat : processStats) {
        Long timeout = 0L;
        if (processStat.getStageStats().size() > 0) {
            timeout = stageEnd - processStat.getStageStats().get(0).getStartTime();
        }
        processTime.put(processStat.getProcessId(), timeout);
    }
    // 获取下mainstem状态信息
    MainStemEventData mainstemData = arbitrateViewService.mainstemData(pipeline.getChannelId(), pipelineId);
    PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
    ChannelStatus status = channelArbitrateEvent.status(pipeline.getChannelId());
    context.put("pipeline", pipeline);
    context.put("pipelineId", pipelineId);
    context.put("processStats", processStats);
    context.put("offset", offset);
    context.put("stageStart", stageStart);
    context.put("stageEnd", stageEnd);
    context.put("processTime", processTime);
    context.put("mainstemData", mainstemData);
    context.put("positionData", positionData);
    context.put("channelStatus", status);
}

Example 5 with PositionEventData

use of com.alibaba.otter.shared.arbitrate.model.PositionEventData in project otter by alibaba.

the class ArbitrateViewServiceImpl method getCanalCursor.

public PositionEventData getCanalCursor(String destination, short clientId) {
    String path = String.format(CANAL_CURSOR_PATH, destination, String.valueOf(clientId));
    try {
        IZkConnection connection = zookeeper.getConnection();
        // zkclient会将获取stat信息和正常的操作分开，使用原生的zk进行优化
        ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();
        Stat stat = new Stat();
        byte[] bytes = orginZk.getData(path, false, stat);
        PositionEventData eventData = new PositionEventData();
        eventData.setCreateTime(new Date(stat.getCtime()));
        eventData.setModifiedTime(new Date(stat.getMtime()));
        eventData.setPosition(new String(bytes, "UTF-8"));
        return eventData;
    } catch (Exception e) {
        return null;
    }
}