Examples with Pipeline com.alibaba.otter.shared.common.model.config.pipeline.Pipeline used on opensource projects

Example 96 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class ChannelArbitrateEvent method canStart.

private boolean canStart(Channel channel) {
    // 判断机器节点是否有存活的通路
    // 查询一下最新的存活的node列表，可能channel取出来的数据为cache的结果
    List<Long> liveNodes = nodeEvent.liveNodes();
    for (Pipeline pipeline : channel.getPipelines()) {
        // 判断select
        List<Long> nids = getNids(pipeline.getSelectNodes());
        if (!CollectionUtils.containsAny(liveNodes, nids)) {
            logger.error("current live nodes:{} , but select nids:{} , result:{}", new Object[] { liveNodes, nids, CollectionUtils.containsAny(liveNodes, nids) });
            sendWarningMessage(pipeline.getId(), "can't restart by no select live node");
            return false;
        }
        // 判断extract
        nids = getNids(pipeline.getExtractNodes());
        if (!CollectionUtils.containsAny(liveNodes, nids)) {
            logger.error("current live nodes:{} , but extract nids:{} , result:{}", new Object[] { liveNodes, nids, CollectionUtils.containsAny(liveNodes, nids) });
            sendWarningMessage(pipeline.getId(), "can't restart by no extract live node");
            return false;
        }
        // 判断transform/load
        nids = getNids(pipeline.getLoadNodes());
        if (!CollectionUtils.containsAny(liveNodes, nids)) {
            logger.error("current live nodes:{} , but transform nids:{} , result:{}", new Object[] { liveNodes, nids, CollectionUtils.containsAny(liveNodes, nids) });
            sendWarningMessage(pipeline.getId(), "can't restart by no transform live node");
            return false;
        }
        // 判断当前没有未清理的process
        List<ProcessStat> stats = arbitrateViewService.listProcesses(channel.getId(), pipeline.getId());
        if (!stats.isEmpty() && !status(channel.getId()).isStart()) {
            List<Long> processIds = new ArrayList<Long>();
            for (ProcessStat stat : stats) {
                processIds.add(stat.getProcessId());
            }
            sendWarningMessage(pipeline.getId(), "can't restart by exist process[" + StringUtils.join(processIds, ',') + "]");
            return false;
        }
    }
    return true;
}

Example 97 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class LocalFileLoaderActionTest method test_load_file.

@Test
public void test_load_file() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPair(10);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    fileBatch.getFiles().addAll(generatorLocalFileData("fileLoad", 10));
    WeightController controller = new WeightController(1);
    fileLoadAction.load(fileBatch, new File(tmp + File.separator + OTTERLOAD), controller);
    File target = new File(tmp + File.separator + OTTERLOAD + "_loaded/");
    want.number(target.listFiles().length).isEqualTo(10);
    NioUtils.delete(target);
}

Example 98 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class AbstractDelegateArbitrateEvent method chooseMode.

public ArbitrateMode chooseMode(Long pipelineId) {
    Pipeline pipeline = ArbitrateConfigUtils.getPipeline(pipelineId);
    ArbitrateMode arbitrateMode = pipeline.getParameters().getArbitrateMode();
    // 重新计算arbitrateMode
    ArbitrateMode result = null;
    switch(arbitrateMode) {
        case AUTOMATIC:
            // 2. 如果s/e由一组机器，t/l由另一组机器提供服务，则选择rpc模式
            if (containAll(pipeline.getSelectNodes(), pipeline.getExtractNodes()) && containAll(pipeline.getSelectNodes(), pipeline.getLoadNodes())) {
                result = ArbitrateMode.MEMORY;
            } else {
                result = ArbitrateMode.RPC;
            }
            break;
        default:
            result = arbitrateMode;
            break;
    }
    return result;
}

Example 99 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class EditPipeline method execute.

/**
 * 找到单个Channel，用于编辑Channel信息界面加载信息
 *
 * @param channelId
 * @param context
 * @throws WebxException
 */
public void execute(@Param("pipelineId") Long pipelineId, Context context, Navigator nav) throws Exception {
    Channel channel = channelService.findByPipelineId(pipelineId);
    if (channel.getStatus().isStart()) {
        nav.redirectTo(WebConstant.ERROR_FORBIDDEN_Link);
        return;
    }
    Pipeline pipeline = pipelineService.findById(pipelineId);
    context.put("pipeline", pipeline);
    context.put("nodes", nodeService.listAll());
}

Example 100 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PipelineList method execute.

public void execute(@Param("channelId") Long channelId, @Param("pipelineId") Long pipelineId, HttpSession session, Context context) throws Exception {
    Channel channel = channelService.findByIdWithoutColumn(channelId);
    List<Pipeline> pipelines = channel.getPipelines();
    List<Pipeline> tempPipe = new ArrayList<Pipeline>();
    if ((pipelineId != null) && (pipelineId != 0l)) {
        for (Pipeline pipeline : pipelines) {
            if (!pipeline.getId().equals(pipelineId)) {
                tempPipe.add(pipeline);
            }
        }
        pipelines.removeAll(tempPipe);
    }
    Map<Long, DelayStat> delayStats = new HashMap<Long, DelayStat>(pipelines.size(), 1f);
    Map<Long, MainStemEventData> mainstemDatas = new HashMap<Long, MainStemEventData>(pipelines.size(), 1f);
    Map<Long, ThroughputStat> throughputStats = new HashMap<Long, ThroughputStat>(pipelines.size(), 1f);
    Map<Long, List<AlarmRule>> alarmRuleStats = new HashMap<Long, List<AlarmRule>>(pipelines.size(), 1f);
    Map<Long, PositionEventData> positionDatas = new HashMap<Long, PositionEventData>(pipelines.size(), 1f);
    for (Pipeline pipeline : pipelines) {
        DelayStat delayStat = delayStatService.findRealtimeDelayStat(pipeline.getId());
        if (delayStat.getDelayNumber() == null) {
            delayStat.setDelayNumber(0L);
            delayStat.setDelayTime(0L);
            delayStat.setGmtModified(pipeline.getGmtModified());
        }
        delayStats.put(pipeline.getId(), delayStat);
        mainstemDatas.put(pipeline.getId(), arbitrateViewService.mainstemData(channel.getId(), pipeline.getId()));
        ThroughputCondition condition = new ThroughputCondition();
        condition.setPipelineId(pipeline.getId());
        condition.setType(ThroughputType.ROW);
        ThroughputStat throughputStat = throughputStatService.findThroughputStatByPipelineId(condition);
        throughputStats.put(pipeline.getId(), throughputStat);
        List<AlarmRule> alarmRules = alarmRuleService.getAlarmRules(pipeline.getId());
        alarmRuleStats.put(pipeline.getId(), alarmRules);
        PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
        positionDatas.put(pipeline.getId(), positionData);
    }
    context.put("channel", channel);
    context.put("pipelines", pipelines);
    context.put("delayStats", delayStats);
    context.put("throughputStats", throughputStats);
    context.put("alarmRuleStats", alarmRuleStats);
    context.put("mainstemDatas", mainstemDatas);
    context.put("positionDatas", positionDatas);
}