Examples with Pipeline com.alibaba.otter.shared.common.model.config.pipeline.Pipeline used on opensource projects

Example 61 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class ChannelServiceImpl method doToModelWithColumn.

private List<Channel> doToModelWithColumn(List<ChannelDO> channelDos) {
    List<Channel> channels = new ArrayList<Channel>();
    try {
        // 1.将ChannelID单独拿出来
        List<Long> channelIds = new ArrayList<Long>();
        for (ChannelDO channelDo : channelDos) {
            channelIds.add(channelDo.getId());
        }
        Long[] idArray = new Long[channelIds.size()];
        // 拿到所有的Pipeline进行ChannelID过滤，避免重复查询。
        List<Pipeline> pipelines = pipelineService.listByChannelIdsWithoutColumn(channelIds.toArray(idArray));
        SystemParameter systemParameter = systemParameterService.find();
        for (ChannelDO channelDo : channelDos) {
            Channel channel = new Channel();
            channel.setId(channelDo.getId());
            channel.setName(channelDo.getName());
            channel.setDescription(channelDo.getDescription());
            ChannelStatus channelStatus = arbitrateManageService.channelEvent().status(channelDo.getId());
            channel.setStatus(null == channelStatus ? ChannelStatus.STOP : channelStatus);
            channel.setParameters(channelDo.getParameters());
            channel.setGmtCreate(channelDo.getGmtCreate());
            channel.setGmtModified(channelDo.getGmtModified());
            // 遍历，将该Channel节点下的Pipeline提取出来。
            List<Pipeline> subPipelines = new ArrayList<Pipeline>();
            for (Pipeline pipeline : pipelines) {
                if (pipeline.getChannelId().equals(channelDo.getId())) {
                    // 合并PipelineParameter和ChannelParameter
                    PipelineParameter parameter = new PipelineParameter();
                    parameter.merge(systemParameter);
                    parameter.merge(channel.getParameters());
                    // 最后复制pipelineId参数
                    parameter.merge(pipeline.getParameters());
                    pipeline.setParameters(parameter);
                    subPipelines.add(pipeline);
                }
            }
            channel.setPipelines(subPipelines);
            channels.add(channel);
        }
    } catch (Exception e) {
        logger.error("ERROR ## change the channels DO to Model has an exception");
        throw new ManagerException(e);
    }
    return channels;
}

Example 62 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class ChannelServiceImpl method listByPipelineIds.

/**
 * <pre>
 * 根据PipelineID找到对应的Channel
 * 优化设想：
 *    应该通过变长参数达到后期扩展的方便性
 * </pre>
 */
public List<Channel> listByPipelineIds(Long... pipelineIds) {
    List<Channel> channels = new ArrayList<Channel>();
    try {
        List<Pipeline> pipelines = pipelineService.listByIds(pipelineIds);
        List<Long> channelIds = new ArrayList<Long>();
        for (Pipeline pipeline : pipelines) {
            if (!channelIds.contains(pipeline.getChannelId())) {
                channelIds.add(pipeline.getChannelId());
            }
        }
        channels = listByIds(channelIds.toArray(new Long[channelIds.size()]));
    } catch (Exception e) {
        logger.error("ERROR ## list query channel by pipelineIds:" + pipelineIds.toString() + " has an exception!");
        throw new ManagerException(e);
    }
    return channels;
}

Example 63 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class ArbitrateForwardIntegration method setUp.

@BeforeMethod
public void setUp() {
    // mock 配置信息数据
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public Channel getChannelByChannelId(Long channelId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(one));
            pipeline.setExtractNodes(Arrays.asList(one));
            pipeline.setLoadNodes(Arrays.asList(two));
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
        }

        @Mock
        public Pipeline getPipeline(Long pipelineId) {
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(one));
            pipeline.setExtractNodes(Arrays.asList(one));
            pipeline.setLoadNodes(Arrays.asList(two));
            return pipeline;
        }

        @Mock
        public Long getCurrentNid() {
            return oneNid;
        }

        @Mock
        public int getParallelism(Long pipelineId) {
            // 并行度
            return 3;
        }

        @Mock
        public Pipeline getOppositePipeline(Long pipelineId) {
            Pipeline pipeline = new Pipeline();
            pipeline.setId(oppositePipelineId);
            pipeline.setSelectNodes(Arrays.asList(two));
            pipeline.setExtractNodes(Arrays.asList(two));
            pipeline.setLoadNodes(Arrays.asList(one));
            return pipeline;
        }

        @Mock
        public Channel getChannel(Long pipelineId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            Pipeline oppositePipeline = new Pipeline();
            oppositePipeline.setId(oppositePipelineId);
            channel.setPipelines(Arrays.asList(pipeline, oppositePipeline));
            return channel;
        }
    });
    Mockit.setUpMock(ArbitrateCommmunicationClient.class, new Object() {

        @Mock
        public Object callManager(final Event event) {
            // do nothing
            return null;
        }
    });
    zookeeper = getZookeeper();
    one.setId(oneNid);
    two.setId(twoNid);
    nodeEvent = new NodeArbitrateEvent();
    // 创建channel
    channelEvent = new ChannelArbitrateEvent();
    pipelineEvent = new PipelineArbitrateEvent();
    // 创建node节点
    nodeEvent.init(oneNid);
    // 创建pipeline节点
    try {
        channelEvent.init(channelId);
    } catch (Exception e) {
    // ignore
    }
    try {
        pipelineEvent.init(channelId, pipelineId);
    } catch (Exception e) {
    // ignore
    }
    arbitrateEventService = (ArbitrateEventService) getBeanFactory().getBean("arbitrateEventService");
    mainStem = new MainStemServiceDemo();
    autowire(mainStem);
    select = new SelectServiceDemo();
    autowire(select);
    extract = new ExtractServiceDemo();
    autowire(extract);
    transform = new TransformServiceDemo();
    autowire(transform);
    load = new LoadServiceDemo();
    autowire(load);
    view = new ProcessViewDemo();
    autowire(view);
    termin = new TerminProcessDemo();
    autowire(termin);
}

Example 64 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class LoadBalanceFactory method getNextTransformNode.

public static Node getNextTransformNode(Long pipelineId) throws InterruptedException {
    Pipeline pipeline = ArbitrateConfigUtils.getPipeline(pipelineId);
    LoadBanlanceAlgorithm loadBanlanceAlgorithm = pipeline.getParameters().getLbAlgorithm();
    LoadBalance loadbalance = null;
    if (loadBanlanceAlgorithm.isRandom()) {
        loadbalance = ArbitrateFactory.getInstance(pipelineId, TransformRandomLoadBanlance.class);
    } else if (loadBanlanceAlgorithm.isRoundRbin()) {
        loadbalance = ArbitrateFactory.getInstance(pipelineId, TransformRoundRobinLoadBalance.class);
    } else {
        loadbalance = ArbitrateFactory.getInstance(pipelineId, TransformStickLoadBalance.class);
    }
    // 获取下一个处理节点信息
    Node node = loadbalance.next();
    return node;
}

Example 65 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class LoadBalanceFactory method getNextExtractNode.

public static Node getNextExtractNode(Long pipelineId) throws InterruptedException {
    Pipeline pipeline = ArbitrateConfigUtils.getPipeline(pipelineId);
    LoadBanlanceAlgorithm loadBanlanceAlgorithm = pipeline.getParameters().getLbAlgorithm();
    LoadBalance loadbalance = null;
    if (loadBanlanceAlgorithm.isRandom()) {
        loadbalance = ArbitrateFactory.getInstance(pipelineId, ExtractRandomLoadBanlance.class);
    } else if (loadBanlanceAlgorithm.isRoundRbin()) {
        loadbalance = ArbitrateFactory.getInstance(pipelineId, ExtractRoundRobinLoadBalance.class);
    } else {
        loadbalance = ArbitrateFactory.getInstance(pipelineId, ExtractStickLoadBalance.class);
    }
    // 获取下一个处理节点信息
    Node node = loadbalance.next();
    return node;
}