Examples with ChannelStatus com.alibaba.otter.shared.common.model.config.channel.ChannelStatus used on opensource projects

Example 16 with ChannelStatus

use of com.alibaba.otter.shared.common.model.config.channel.ChannelStatus in project otter by alibaba.

the class ChannelServiceImpl method doToModelWithColumn.

private List<Channel> doToModelWithColumn(List<ChannelDO> channelDos) {
    List<Channel> channels = new ArrayList<Channel>();
    try {
        // 1.将ChannelID单独拿出来
        List<Long> channelIds = new ArrayList<Long>();
        for (ChannelDO channelDo : channelDos) {
            channelIds.add(channelDo.getId());
        }
        Long[] idArray = new Long[channelIds.size()];
        // 拿到所有的Pipeline进行ChannelID过滤，避免重复查询。
        List<Pipeline> pipelines = pipelineService.listByChannelIdsWithoutColumn(channelIds.toArray(idArray));
        SystemParameter systemParameter = systemParameterService.find();
        for (ChannelDO channelDo : channelDos) {
            Channel channel = new Channel();
            channel.setId(channelDo.getId());
            channel.setName(channelDo.getName());
            channel.setDescription(channelDo.getDescription());
            ChannelStatus channelStatus = arbitrateManageService.channelEvent().status(channelDo.getId());
            channel.setStatus(null == channelStatus ? ChannelStatus.STOP : channelStatus);
            channel.setParameters(channelDo.getParameters());
            channel.setGmtCreate(channelDo.getGmtCreate());
            channel.setGmtModified(channelDo.getGmtModified());
            // 遍历，将该Channel节点下的Pipeline提取出来。
            List<Pipeline> subPipelines = new ArrayList<Pipeline>();
            for (Pipeline pipeline : pipelines) {
                if (pipeline.getChannelId().equals(channelDo.getId())) {
                    // 合并PipelineParameter和ChannelParameter
                    PipelineParameter parameter = new PipelineParameter();
                    parameter.merge(systemParameter);
                    parameter.merge(channel.getParameters());
                    // 最后复制pipelineId参数
                    parameter.merge(pipeline.getParameters());
                    pipeline.setParameters(parameter);
                    subPipelines.add(pipeline);
                }
            }
            channel.setPipelines(subPipelines);
            channels.add(channel);
        }
    } catch (Exception e) {
        logger.error("ERROR ## change the channels DO to Model has an exception");
        throw new ManagerException(e);
    }
    return channels;
}

Example 17 with ChannelStatus

use of com.alibaba.otter.shared.common.model.config.channel.ChannelStatus in project otter by alibaba.

the class GlobalMonitor method concurrentProcess.

private void concurrentProcess(List<Long> channelIds) {
    ExecutorCompletionService completionExecutor = new ExecutorCompletionService(executor);
    List<Future> futures = new ArrayList<Future>();
    for (final Long channelId : channelIds) {
        futures.add(completionExecutor.submit(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                ChannelStatus status = arbitrateManageService.channelEvent().status(channelId);
                if (status.isPause()) {
                    restartAlarmRecovery.recovery(channelId);
                }
                return null;
            }
        }));
    }
    List<Throwable> exceptions = new ArrayList<Throwable>();
    int index = 0;
    int size = futures.size();
    while (index < size) {
        try {
            Future<?> future = completionExecutor.take();
            future.get();
        } catch (InterruptedException e) {
            exceptions.add(e);
        } catch (ExecutionException e) {
            exceptions.add(e);
        }
        index++;
    }
    if (!exceptions.isEmpty()) {
        StringBuilder sb = new StringBuilder(exceptions.size() + " exception happens in global monitor\n");
        sb.append("exception stack start :\n");
        for (Throwable t : exceptions) {
            sb.append(ExceptionUtils.getStackTrace(t));
        }
        sb.append("exception stack end \n");
        throw new IllegalStateException(sb.toString());
    }
}

Example 18 with ChannelStatus

use of com.alibaba.otter.shared.common.model.config.channel.ChannelStatus in project otter by alibaba.

the class ChannelList method execute.

public void execute(@Param("pageIndex") int pageIndex, @Param("searchKey") String searchKey, @Param("channelStatus") String status, @Param("channelId") Long channelId, @Param("errorType") String errorType, Context context) throws Exception {
    @SuppressWarnings("unchecked") Map<String, Object> condition = new HashMap<String, Object>();
    if ("请输入关键字(目前支持Channel的ID、名字搜索)".equals(searchKey)) {
        searchKey = "";
    }
    condition.put("searchKey", searchKey);
    List<Long> theStatusPks = new ArrayList<Long>();
    if (null != status) {
        List<Long> allChannelPks = channelService.listAllChannelId();
        for (Long channelPk : allChannelPks) {
            ChannelStatus channelStatus = arbitrateManageService.channelEvent().status(channelPk);
            if (channelStatus.equals(ChannelStatus.valueOf(status))) {
                theStatusPks.add(channelPk);
            }
        }
    }
    int count = channelService.getCount(condition);
    Paginator paginator = new Paginator();
    paginator.setItems(count);
    paginator.setPage(pageIndex);
    condition.put("offset", paginator.getOffset());
    condition.put("length", paginator.getLength());
    List<Channel> channels = new ArrayList<Channel>();
    if ((null != channelId) && (channelId != 0l)) {
        channels.add(channelService.findById(channelId));
        paginator.setItems(1);
        paginator.setPage(0);
        // 定义为新的searchKey
        searchKey = String.valueOf(channelId);
    } else {
        channels = channelService.listByConditionWithoutColumn(condition);
    }
    List<SeniorChannel> seniorChannels = new ArrayList<SeniorChannel>();
    for (Channel channel : channels) {
        boolean processEmpty = false;
        List<Pipeline> pipelines = channel.getPipelines();
        for (Pipeline pipeline : pipelines) {
            if (processStatService.listRealtimeProcessStat(channel.getId(), pipeline.getId()).isEmpty()) {
                processEmpty = true;
            }
        }
        SeniorChannel seniorChannel = new SeniorChannel();
        seniorChannel.setId(channel.getId());
        seniorChannel.setName(channel.getName());
        seniorChannel.setParameters(channel.getParameters());
        seniorChannel.setPipelines(channel.getPipelines());
        seniorChannel.setStatus(channel.getStatus());
        seniorChannel.setDescription(channel.getDescription());
        seniorChannel.setGmtCreate(channel.getGmtCreate());
        seniorChannel.setGmtModified(channel.getGmtModified());
        seniorChannel.setProcessEmpty(processEmpty);
        seniorChannels.add(seniorChannel);
    }
    context.put("channels", seniorChannels);
    context.put("paginator", paginator);
    context.put("searchKey", searchKey);
    context.put("errorType", errorType);
}

Example 19 with ChannelStatus

use of com.alibaba.otter.shared.common.model.config.channel.ChannelStatus in project otter by alibaba.

the class AnalysisStageStat method execute.

public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {
    List<ProcessStat> processStats = new ArrayList<ProcessStat>();
    Pipeline pipeline = pipelineService.findById(pipelineId);
    processStats = processStatService.listRealtimeProcessStat(pipelineId);
    // Map ext = new HashMap<Long, String>();
    // // ext.put(145456451, "asdf");
    // for (Long i = 1L; i <= 3; i++) {
    // List<StageStat> stageStats = new ArrayList<StageStat>();
    // ProcessStat processStat = new ProcessStat();
    // processStat.setPipelineId(1L);
    // processStat.setProcessId(i);
    // StageStat stage = new StageStat();
    // stage.setStage(StageType.SELECT);
    // stage.setStartTime(((new Date()).getTime() + i * 10 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 200 * 1000));
    // stage.setNumber(11231230L);
    // stage.setSize(14545645640L);
    // // stage.setExts(ext);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.EXTRACT);
    // stage.setStartTime(((new Date()).getTime() + i * 2000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 3000 * 1000));
    // stage.setExts(ext);
    // // stage.setNumber(10L);
    // // stage.setSize(10L);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.TRANSFORM);
    // stage.setStartTime(((new Date()).getTime() + i * 5000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 6000 * 1000));
    // stage.setNumber(154640L);
    // stage.setExts(ext);
    // // stage.setSize(10L);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.LOAD);
    // stage.setStartTime(((new Date()).getTime() + i * 70000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 80000 * 1000));
    // // stage.setNumber(10L);
    // stage.setSize(101445L);
    // // stage.setExts(ext);
    // stageStats.add(stage);
    // processStat.setStageStats(stageStats);
    // processStats.add(processStat);
    // }
    Long stageStart = 0L;
    // Long stageEnd = new Date().getTime() + 3 * 80000 * 1000;
    Long stageEnd = new Date().getTime();
    Long interval = 0L;
    double offset = 0L;
    // 找出最先开始的process的select阶段的开始时间作为起始时间
    if (processStats.size() > 0) {
        if (processStats.get(0).getStageStats().size() > 0) {
            stageStart = processStats.get(0).getStageStats().get(0).getStartTime();
        }
    }
    // 动态计算每个阶段的长度比例
    if (stageStart > 0) {
        interval = stageEnd - stageStart;
    }
    if (interval > 0) {
        offset = 800.0 / interval;
    }
    // 计算每个process当前任务所做的时间总和
    Map<Long, Long> processTime = new HashMap<Long, Long>();
    for (ProcessStat processStat : processStats) {
        Long timeout = 0L;
        if (processStat.getStageStats().size() > 0) {
            timeout = stageEnd - processStat.getStageStats().get(0).getStartTime();
        }
        processTime.put(processStat.getProcessId(), timeout);
    }
    // 获取下mainstem状态信息
    MainStemEventData mainstemData = arbitrateViewService.mainstemData(pipeline.getChannelId(), pipelineId);
    PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
    ChannelStatus status = channelArbitrateEvent.status(pipeline.getChannelId());
    context.put("pipeline", pipeline);
    context.put("pipelineId", pipelineId);
    context.put("processStats", processStats);
    context.put("offset", offset);
    context.put("stageStart", stageStart);
    context.put("stageEnd", stageEnd);
    context.put("processTime", processTime);
    context.put("mainstemData", mainstemData);
    context.put("positionData", positionData);
    context.put("channelStatus", status);
}

Example 20 with ChannelStatus

use of com.alibaba.otter.shared.common.model.config.channel.ChannelStatus in project otter by alibaba.

the class SelectRpcArbitrateEvent method await.

public EtlEventData await(Long pipelineId) throws InterruptedException {
    Assert.notNull(pipelineId);
    PermitMonitor permitMonitor = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
    // 阻塞等待授权
    permitMonitor.waitForPermit();
    SelectProcessListener selectProcessListener = ArbitrateFactory.getInstance(pipelineId, SelectProcessListener.class);
    // 符合条件的processId
    Long processId = selectProcessListener.waitForProcess();
    ChannelStatus status = permitMonitor.getChannelPermit();
    if (status.isStart()) {
        // 即时查询一下当前的状态，状态随时可能会变
        try {
            EtlEventData eventData = new EtlEventData();
            eventData.setPipelineId(pipelineId);
            eventData.setProcessId(processId);
            // 返回当前时间
            eventData.setStartTime(new Date().getTime());
            // 获取下一个处理节点信息
            Node node = LoadBalanceFactory.getNextExtractNode(pipelineId);
            if (node == null) {
                // terminEvent.single(termin);
                throw new ArbitrateException("Select_single", "no next node");
            } else {
                eventData.setNextNid(node.getId());
                // 标记为已使用
                markUsed(eventData);
                // 只有这一条路返回
                return eventData;
            }
        } catch (ZkNoNodeException e) {
            logger.error("pipeline[{}] processId[{}] is invalid , retry again", pipelineId, processId);
            // /出现节点不存在，说明出现了error情况,递归调用重新获取一次
            return await(pipelineId);
        } catch (ZkException e) {
            throw new ArbitrateException("Select_await", e.getMessage(), e);
        }
    } else {
        logger.warn("pipelineId[{}] select ignore processId[{}] by status[{}]", new Object[] { pipelineId, processId, status });
        // add by ljh 2013-02-01
        // 遇到一个bug:
        // a. 某台机器发起了一个RESTART指令，然后开始删除process列表
        // b. 此时另一个台机器(select工作节点)，并没有收到PAUSE的推送，导致还会再创建一个process节点
        // c. 后续收到PAUSE指令后，丢弃了processId，就出现了unused的processId
        // 这里删除了，要考虑一个问题，就是和restart指令在并行删除同一个processId时的并发考虑，目前来看没问题
        String path = StagePathUtils.getProcess(pipelineId, processId);
        // 忽略删除失败
        zookeeper.delete(path);
        // 递归调用
        return await(pipelineId);
    }
}