Examples with MainStemEventData com.alibaba.otter.shared.arbitrate.model.MainStemEventData used on opensource projects

Example 6 with MainStemEventData

use of com.alibaba.otter.shared.arbitrate.model.MainStemEventData in project otter by alibaba.

the class OtterDownStreamHandler method notifyMainstemStatus.

private void notifyMainstemStatus(MainStemEventData.Status status) {
    MainStemEventData mainStemData = new MainStemEventData();
    mainStemData.setPipelineId(pipelineId);
    mainStemData.setStatus(status);
    arbitrateEventService.mainStemEvent().single(mainStemData);
}

Example 7 with MainStemEventData

use of com.alibaba.otter.shared.arbitrate.model.MainStemEventData in project otter by alibaba.

the class AnalysisTopStat method execute.

public void execute(@Param("searchKey") String searchKey, @Param("topN") int topN, @Param("statTime") int minute, Context context) throws Exception {
    if (topN <= 0) {
        topN = 15;
    }
    if (minute <= 0) {
        minute = 1;
    }
    List<TopDelayStat> tops = delayStatService.listTopDelayStat(searchKey, topN);
    List<Long> pipelineIds = new ArrayList<Long>();
    for (TopDelayStat top : tops) {
        top.setStatTime(Long.valueOf(minute));
        pipelineIds.add(top.getPipelineId());
    }
    Map<Long, ChannelStatus> channelStatuses = new HashMap<Long, ChannelStatus>(tops.size(), 1f);
    Map<Long, MainStemEventData> mainstemStatuses = new HashMap<Long, MainStemEventData>(tops.size(), 1f);
    if (pipelineIds.size() > 0) {
        List<ThroughputStat> stats = throughputStatService.listRealtimeThroughputByPipelineIds(pipelineIds, minute);
        for (ThroughputStat stat : stats) {
            for (TopDelayStat top : tops) {
                if (stat.getPipelineId().equals(top.getPipelineId())) {
                    DataStat s = new DataStat(stat.getNumber(), stat.getSize());
                    if (ThroughputType.FILE == stat.getType()) {
                        top.setFileStat(s);
                    } else if (ThroughputType.ROW == stat.getType()) {
                        top.setDbStat(s);
                    }
                    break;
                }
            }
        }
        for (TopDelayStat top : tops) {
            if (!channelStatuses.containsKey(top.getChannelId())) {
                channelStatuses.put(top.getChannelId(), arbitrateManageService.channelEvent().status(top.getChannelId()));
            }
            if (!mainstemStatuses.containsKey(top.getPipelineId())) {
                mainstemStatuses.put(top.getPipelineId(), arbitrateViewService.mainstemData(top.getChannelId(), top.getPipelineId()));
            }
        }
    }
    context.put("tops", tops);
    context.put("statTime", minute);
    context.put("channelStatuses", channelStatuses);
    context.put("mainstemStatuses", mainstemStatuses);
}

Example 8 with MainStemEventData

use of com.alibaba.otter.shared.arbitrate.model.MainStemEventData in project otter by alibaba.

the class BaseStageTest method init.

@BeforeClass
public void init() {
    // 初始化节点
    // mock 配置信息数据
    local.setStatus(NodeStatus.START);
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public Channel getChannelByChannelId(Long channelId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
        }

        @Mock
        public Channel getChannel(Long pipelineId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
        }

        @Mock
        public Pipeline getPipeline(Long pipelineId) {
            Pipeline pipeline = new Pipeline();
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            return pipeline;
        }

        @Mock
        public Pipeline getOppositePipeline(Long pipelineId) {
            // 没有反向同步
            return null;
        }

        @Mock
        public Long getCurrentNid() {
            return nid;
        }

        @Mock
        public int getParallelism(Long pipelineId) {
            // 并行度
            return 3;
        }
    });
    Mockit.setUpMock(ArbitrateCommmunicationClient.class, new Object() {

        @Mock
        public Object callManager(final Event event) {
            // do nothing
            return null;
        }

        @Mock
        public void callManager(final Event event, final Callback callback) {
        // do nothing
        }
    });
    zookeeper = getZookeeper();
    local.setId(nid);
    nodeEvent = new NodeArbitrateEvent();
    channelEvent = new ChannelArbitrateEvent();
    pipelineEvent = new PipelineArbitrateEvent();
    pipelinePath = StagePathUtils.getPipeline(channelId, pipelineId);
    processPath = StagePathUtils.getProcessRoot(channelId, pipelineId);
    channelEvent.init(channelId);
    pipelineEvent.init(channelId, pipelineId);
    channelEvent.start(channelId);
    String path = pipelinePath + "/" + ArbitrateConstants.NODE_MAINSTEM;
    MainStemEventData eventData = new MainStemEventData();
    eventData.setStatus(MainStemEventData.Status.OVERTAKE);
    eventData.setNid(nid);
    // 初始化的数据对象
    byte[] bytes = JsonUtils.marshalToByte(eventData);
    zookeeper.create(path, bytes, CreateMode.EPHEMERAL);
}

Example 9 with MainStemEventData

use of com.alibaba.otter.shared.arbitrate.model.MainStemEventData in project otter by alibaba.

the class PipelineList method execute.

public void execute(@Param("channelId") Long channelId, @Param("pipelineId") Long pipelineId, HttpSession session, Context context) throws Exception {
    Channel channel = channelService.findByIdWithoutColumn(channelId);
    List<Pipeline> pipelines = channel.getPipelines();
    List<Pipeline> tempPipe = new ArrayList<Pipeline>();
    if ((pipelineId != null) && (pipelineId != 0l)) {
        for (Pipeline pipeline : pipelines) {
            if (!pipeline.getId().equals(pipelineId)) {
                tempPipe.add(pipeline);
            }
        }
        pipelines.removeAll(tempPipe);
    }
    Map<Long, DelayStat> delayStats = new HashMap<Long, DelayStat>(pipelines.size(), 1f);
    Map<Long, MainStemEventData> mainstemDatas = new HashMap<Long, MainStemEventData>(pipelines.size(), 1f);
    Map<Long, ThroughputStat> throughputStats = new HashMap<Long, ThroughputStat>(pipelines.size(), 1f);
    Map<Long, List<AlarmRule>> alarmRuleStats = new HashMap<Long, List<AlarmRule>>(pipelines.size(), 1f);
    Map<Long, PositionEventData> positionDatas = new HashMap<Long, PositionEventData>(pipelines.size(), 1f);
    for (Pipeline pipeline : pipelines) {
        DelayStat delayStat = delayStatService.findRealtimeDelayStat(pipeline.getId());
        if (delayStat.getDelayNumber() == null) {
            delayStat.setDelayNumber(0L);
            delayStat.setDelayTime(0L);
            delayStat.setGmtModified(pipeline.getGmtModified());
        }
        delayStats.put(pipeline.getId(), delayStat);
        mainstemDatas.put(pipeline.getId(), arbitrateViewService.mainstemData(channel.getId(), pipeline.getId()));
        ThroughputCondition condition = new ThroughputCondition();
        condition.setPipelineId(pipeline.getId());
        condition.setType(ThroughputType.ROW);
        ThroughputStat throughputStat = throughputStatService.findThroughputStatByPipelineId(condition);
        throughputStats.put(pipeline.getId(), throughputStat);
        List<AlarmRule> alarmRules = alarmRuleService.getAlarmRules(pipeline.getId());
        alarmRuleStats.put(pipeline.getId(), alarmRules);
        PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
        positionDatas.put(pipeline.getId(), positionData);
    }
    context.put("channel", channel);
    context.put("pipelines", pipelines);
    context.put("delayStats", delayStats);
    context.put("throughputStats", throughputStats);
    context.put("alarmRuleStats", alarmRuleStats);
    context.put("mainstemDatas", mainstemDatas);
    context.put("positionDatas", positionDatas);
}

Example 10 with MainStemEventData

use of com.alibaba.otter.shared.arbitrate.model.MainStemEventData in project otter by alibaba.

the class AnalysisStageStat method execute.

public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {
    List<ProcessStat> processStats = new ArrayList<ProcessStat>();
    Pipeline pipeline = pipelineService.findById(pipelineId);
    processStats = processStatService.listRealtimeProcessStat(pipelineId);
    // Map ext = new HashMap<Long, String>();
    // // ext.put(145456451, "asdf");
    // for (Long i = 1L; i <= 3; i++) {
    // List<StageStat> stageStats = new ArrayList<StageStat>();
    // ProcessStat processStat = new ProcessStat();
    // processStat.setPipelineId(1L);
    // processStat.setProcessId(i);
    // StageStat stage = new StageStat();
    // stage.setStage(StageType.SELECT);
    // stage.setStartTime(((new Date()).getTime() + i * 10 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 200 * 1000));
    // stage.setNumber(11231230L);
    // stage.setSize(14545645640L);
    // // stage.setExts(ext);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.EXTRACT);
    // stage.setStartTime(((new Date()).getTime() + i * 2000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 3000 * 1000));
    // stage.setExts(ext);
    // // stage.setNumber(10L);
    // // stage.setSize(10L);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.TRANSFORM);
    // stage.setStartTime(((new Date()).getTime() + i * 5000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 6000 * 1000));
    // stage.setNumber(154640L);
    // stage.setExts(ext);
    // // stage.setSize(10L);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.LOAD);
    // stage.setStartTime(((new Date()).getTime() + i * 70000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 80000 * 1000));
    // // stage.setNumber(10L);
    // stage.setSize(101445L);
    // // stage.setExts(ext);
    // stageStats.add(stage);
    // processStat.setStageStats(stageStats);
    // processStats.add(processStat);
    // }
    Long stageStart = 0L;
    // Long stageEnd = new Date().getTime() + 3 * 80000 * 1000;
    Long stageEnd = new Date().getTime();
    Long interval = 0L;
    double offset = 0L;
    // 找出最先开始的process的select阶段的开始时间作为起始时间
    if (processStats.size() > 0) {
        if (processStats.get(0).getStageStats().size() > 0) {
            stageStart = processStats.get(0).getStageStats().get(0).getStartTime();
        }
    }
    // 动态计算每个阶段的长度比例
    if (stageStart > 0) {
        interval = stageEnd - stageStart;
    }
    if (interval > 0) {
        offset = 800.0 / interval;
    }
    // 计算每个process当前任务所做的时间总和
    Map<Long, Long> processTime = new HashMap<Long, Long>();
    for (ProcessStat processStat : processStats) {
        Long timeout = 0L;
        if (processStat.getStageStats().size() > 0) {
            timeout = stageEnd - processStat.getStageStats().get(0).getStartTime();
        }
        processTime.put(processStat.getProcessId(), timeout);
    }
    // 获取下mainstem状态信息
    MainStemEventData mainstemData = arbitrateViewService.mainstemData(pipeline.getChannelId(), pipelineId);
    PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
    ChannelStatus status = channelArbitrateEvent.status(pipeline.getChannelId());
    context.put("pipeline", pipeline);
    context.put("pipelineId", pipelineId);
    context.put("processStats", processStats);
    context.put("offset", offset);
    context.put("stageStart", stageStart);
    context.put("stageEnd", stageEnd);
    context.put("processTime", processTime);
    context.put("mainstemData", mainstemData);
    context.put("positionData", positionData);
    context.put("channelStatus", status);
}