Examples with Pipeline com.alibaba.otter.shared.common.model.config.pipeline.Pipeline used on opensource projects

Example 26 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class CanalEmbedSelector method start.

public void start() {
    if (running) {
        return;
    }
    // 获取destination/filter参数
    Pipeline pipeline = configClientService.findPipeline(pipelineId);
    filter = CanalFilterSupport.makeFilterExpression(pipeline);
    destination = pipeline.getParameters().getDestinationName();
    batchSize = pipeline.getParameters().getMainstemBatchsize();
    batchTimeout = pipeline.getParameters().getBatchTimeout();
    ddlSync = pipeline.getParameters().getDdlSync();
    final boolean syncFull = pipeline.getParameters().getSyncMode().isRow() || pipeline.getParameters().isEnableRemedy();
    // 暂时使用skip load代替
    filterTableError = pipeline.getParameters().getSkipSelectException();
    if (pipeline.getParameters().getDumpSelector() != null) {
        dump = pipeline.getParameters().getDumpSelector();
    }
    if (pipeline.getParameters().getDumpSelectorDetail() != null) {
        dumpDetail = pipeline.getParameters().getDumpSelectorDetail();
    }
    canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

        public CanalInstance generate(String destination) {
            Canal canal = canalConfigClient.findCanal(destination);
            final OtterAlarmHandler otterAlarmHandler = new OtterAlarmHandler();
            otterAlarmHandler.setPipelineId(pipelineId);
            // 注入一下spring资源
            OtterContextLocator.autowire(otterAlarmHandler);
            // 设置下slaveId，保证多个piplineId下重复引用时不重复
            // 默认基数
            long slaveId = 10000;
            if (canal.getCanalParameter().getSlaveId() != null) {
                slaveId = canal.getCanalParameter().getSlaveId();
            }
            canal.getCanalParameter().setSlaveId(slaveId + pipelineId);
            canal.getCanalParameter().setDdlIsolation(ddlSync);
            canal.getCanalParameter().setFilterTableError(filterTableError);
            canal.getCanalParameter().setMemoryStorageRawEntry(false);
            CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, filter) {

                protected CanalHAController initHaController() {
                    HAMode haMode = parameters.getHaMode();
                    if (haMode.isMedia()) {
                        return new MediaHAController(parameters.getMediaGroup(), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName());
                    } else {
                        return super.initHaController();
                    }
                }

                protected void startEventParserInternal(CanalEventParser parser, boolean isGroup) {
                    super.startEventParserInternal(parser, isGroup);
                    if (eventParser instanceof MysqlEventParser) {
                        // 设置支持的类型
                        ((MysqlEventParser) eventParser).setSupportBinlogFormats("ROW");
                        if (syncFull) {
                            ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL");
                        } else {
                            ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL,MINIMAL");
                        }
                        MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
                        // otter先使用简单的模式
                        mysqlEventParser.setParallel(false);
                        CanalHAController haController = mysqlEventParser.getHaController();
                        if (haController instanceof MediaHAController) {
                            if (isGroup) {
                                throw new CanalException("not support group database use media HA");
                            }
                            ((MediaHAController) haController).setCanalHASwitchable(mysqlEventParser);
                        }
                        if (!haController.isStart()) {
                            haController.start();
                        }
                        // 基于media的Ha，直接从tddl中获取数据库信息
                        if (haController instanceof MediaHAController) {
                            AuthenticationInfo authenticationInfo = ((MediaHAController) haController).getAvailableAuthenticationInfo();
                            ((MysqlEventParser) eventParser).setMasterInfo(authenticationInfo);
                        }
                    }
                }
            };
            instance.setAlarmHandler(otterAlarmHandler);
            CanalEventSink eventSink = instance.getEventSink();
            if (eventSink instanceof AbstractCanalEventSink) {
                handler = new OtterDownStreamHandler();
                handler.setPipelineId(pipelineId);
                handler.setDetectingIntervalInSeconds(canal.getCanalParameter().getDetectingIntervalInSeconds());
                // 注入一下spring资源
                OtterContextLocator.autowire(handler);
                // 添加到开头
                ((AbstractCanalEventSink) eventSink).addHandler(handler, 0);
                handler.start();
            }
            return instance;
        }
    });
    canalServer.start();
    canalServer.start(destination);
    this.clientIdentity = new ClientIdentity(destination, pipeline.getParameters().getMainstemClientId(), filter);
    // 发起一次订阅
    canalServer.subscribe(clientIdentity);
    running = true;
}

Example 27 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class BaseStageTest method init.

@BeforeClass
public void init() {
    // 初始化节点
    // mock 配置信息数据
    local.setStatus(NodeStatus.START);
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public Channel getChannelByChannelId(Long channelId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
        }

        @Mock
        public Channel getChannel(Long pipelineId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
        }

        @Mock
        public Pipeline getPipeline(Long pipelineId) {
            Pipeline pipeline = new Pipeline();
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            return pipeline;
        }

        @Mock
        public Pipeline getOppositePipeline(Long pipelineId) {
            // 没有反向同步
            return null;
        }

        @Mock
        public Long getCurrentNid() {
            return nid;
        }

        @Mock
        public int getParallelism(Long pipelineId) {
            // 并行度
            return 3;
        }
    });
    Mockit.setUpMock(ArbitrateCommmunicationClient.class, new Object() {

        @Mock
        public Object callManager(final Event event) {
            // do nothing
            return null;
        }

        @Mock
        public void callManager(final Event event, final Callback callback) {
        // do nothing
        }
    });
    zookeeper = getZookeeper();
    local.setId(nid);
    nodeEvent = new NodeArbitrateEvent();
    channelEvent = new ChannelArbitrateEvent();
    pipelineEvent = new PipelineArbitrateEvent();
    pipelinePath = StagePathUtils.getPipeline(channelId, pipelineId);
    processPath = StagePathUtils.getProcessRoot(channelId, pipelineId);
    channelEvent.init(channelId);
    pipelineEvent.init(channelId, pipelineId);
    channelEvent.start(channelId);
    String path = pipelinePath + "/" + ArbitrateConstants.NODE_MAINSTEM;
    MainStemEventData eventData = new MainStemEventData();
    eventData.setStatus(MainStemEventData.Status.OVERTAKE);
    eventData.setNid(nid);
    // 初始化的数据对象
    byte[] bytes = JsonUtils.marshalToByte(eventData);
    zookeeper.create(path, bytes, CreateMode.EPHEMERAL);
}

Example 28 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class ArbitrateAllTest method setUp.

@BeforeMethod
public void setUp() {
    // mock 配置信息数据
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public Channel getChannelByChannelId(Long channelId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            pipeline.getParameters().setArbitrateMode(arbitrateMode);
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
        }

        @Mock
        public Pipeline getPipeline(Long pipelineId) {
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            pipeline.getParameters().setArbitrateMode(arbitrateMode);
            return pipeline;
        }

        @Mock
        public Long getCurrentNid() {
            return nid;
        }

        @Mock
        public int getParallelism(Long pipelineId) {
            // 并行度
            return 3;
        }

        @Mock
        public Pipeline getOppositePipeline(Long pipelineId) {
            return null;
        }

        @Mock
        public Channel getChannel(Long pipelineId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.getParameters().setArbitrateMode(arbitrateMode);
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
        }
    });
    Mockit.setUpMock(ArbitrateCommmunicationClient.class, new Object() {

        @Mock
        public Object callManager(final Event event) {
            // do nothing
            return null;
        }
    });
    zookeeper = getZookeeper();
    local.setId(nid);
    nodeEvent = new NodeArbitrateEvent();
    channelEvent = new ChannelArbitrateEvent();
    pipelineEvent = new PipelineArbitrateEvent();
    // 创建node节点
    nodeEvent.init(local.getId());
    // 创建pipeline节点
    channelEvent.init(channelId);
    pipelineEvent.init(channelId, pipelineId);
    arbitrateEventService = (ArbitrateEventService) getBeanFactory().getBean("arbitrateEventService");
    mainStem = new MainStemServiceDemo();
    autowire(mainStem);
    select = new SelectServiceDemo();
    autowire(select);
    extract = new ExtractServiceDemo();
    autowire(extract);
    transform = new TransformServiceDemo();
    autowire(transform);
    load = new LoadServiceDemo();
    autowire(load);
    view = new ProcessViewDemo();
    autowire(view);
    termin = new TerminProcessDemo();
    autowire(termin);
}

Example 29 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class RandomLoadBalanceTest method testTransform.

@Test
public void testTransform() {
    // 初始化节点
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public Pipeline getPipeline(Long pipelineId) {
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(sourceList);
            pipeline.setExtractNodes(sourceList);
            pipeline.setLoadNodes(targetList);
            return pipeline;
        }
    });
    TransformRandomLoadBanlance transform = new TransformRandomLoadBanlance(pipelineId);
    transform.setNodeMonitor(nodeMonitor);
    sleep(500L);
    try {
        Node n1 = transform.next();
        Node n2 = transform.next();
        Node n3 = transform.next();
        Node n4 = transform.next();
        System.out.printf("n1[%s] n2[%s] n3[%s] n4[%s]", n1.getId(), n2.getId(), n3.getId(), n4.getId());
        want.bool(targetList.contains(n1)).is(true);
        want.bool(targetList.contains(n2)).is(true);
        want.bool(targetList.contains(n3)).is(true);
        want.bool(targetList.contains(n4)).is(true);
    } catch (InterruptedException e) {
        want.fail();
    }
}

Example 30 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class RoundRobinBalanceTest method testExtract.

@Test
public void testExtract() {
    // 初始化节点
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public Pipeline getPipeline(Long pipelineId) {
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(sourceList);
            pipeline.setExtractNodes(sourceList);
            pipeline.setLoadNodes(targetList);
            return pipeline;
        }
    });
    ExtractRoundRobinLoadBalance extract = new ExtractRoundRobinLoadBalance(pipelineId);
    extract.setNodeMonitor(nodeMonitor);
    try {
        Node n1 = extract.next();
        Node n2 = extract.next();
        Node n3 = extract.next();
        Node n4 = extract.next();
        System.out.printf("n1[%s] n2[%s] n3[%s] n4[%s]", n1.getId(), n2.getId(), n3.getId(), n4.getId());
        want.bool(sourceList.contains(n1)).is(true);
        want.bool(sourceList.contains(n2)).is(true);
        want.bool(sourceList.contains(n3)).is(true);
        want.bool(sourceList.contains(n4)).is(true);
    } catch (InterruptedException e) {
        want.fail();
    }
}