Examples with Canal com.alibaba.otter.canal.instance.manager.model.Canal used on opensource projects

Example 6 with Canal

use of com.alibaba.otter.canal.instance.manager.model.Canal in project otter by alibaba.

the class CanalAction method doAdd.

/**
     * 添加canal
     */
public void doAdd(@FormGroup("canalInfo") Group canalInfo, @FormGroup("canalParameterInfo") Group canalParameterInfo, @FormField(name = "formCanalError", group = "canalInfo") CustomErrors err, @FormField(name = "formHeartBeatError", group = "canalParameterInfo") CustomErrors heartBeatErr, Navigator nav) throws Exception {
    Canal canal = new Canal();
    CanalParameter parameter = new CanalParameter();
    canalInfo.setProperties(canal);
    canalParameterInfo.setProperties(parameter);
    String zkClustersString = canalParameterInfo.getField("zkClusters").getStringValue();
    String[] zkClusters = StringUtils.split(zkClustersString, ";");
    parameter.setZkClusters(Arrays.asList(zkClusters));
    Long zkClusterId = canalParameterInfo.getField("autoKeeperClusterId").getLongValue();
    parameter.setZkClusterId(zkClusterId);
    canal.setCanalParameter(parameter);
    String dbAddressesString = canalParameterInfo.getField("groupDbAddresses").getStringValue();
    // 第一层的分号代表主备概念，,第二层逗号代表分组概念
    if (StringUtils.isNotEmpty(dbAddressesString)) {
        List<List<DataSourcing>> dbSocketAddress = new ArrayList<List<DataSourcing>>();
        String[] dbAddresses = StringUtils.split(dbAddressesString, ";");
        for (String dbAddressString : dbAddresses) {
            List<DataSourcing> groupDbSocketAddress = new ArrayList<DataSourcing>();
            String[] groupDbAddresses = StringUtils.split(dbAddressString, ",");
            for (String groupDbAddress : groupDbAddresses) {
                String[] strs = StringUtils.split(groupDbAddress, ":");
                InetSocketAddress address = new InetSocketAddress(strs[0].trim(), Integer.valueOf(strs[1]));
                SourcingType type = parameter.getSourcingType();
                if (strs.length > 2) {
                    type = SourcingType.valueOf(strs[2]);
                }
                groupDbSocketAddress.add(new DataSourcing(type, address));
            }
            dbSocketAddress.add(groupDbSocketAddress);
        }
        parameter.setGroupDbAddresses(dbSocketAddress);
    }
    String positionsString = canalParameterInfo.getField("positions").getStringValue();
    if (StringUtils.isNotEmpty(positionsString)) {
        String[] positions = StringUtils.split(positionsString, ";");
        parameter.setPositions(Arrays.asList(positions));
    }
    if (parameter.getDetectingEnable() && StringUtils.startsWithIgnoreCase(parameter.getDetectingSQL(), "select")) {
        heartBeatErr.setMessage("invaliedHeartBeat");
        return;
    }
    try {
        canalService.create(canal);
    } catch (RepeatConfigureException rce) {
        err.setMessage("invalidCanal");
        return;
    }
    if (parameter.getSourcingType().isMysql() && parameter.getSlaveId() == null) {
        parameter.setSlaveId(10000 + canal.getId());
        // 再次更新一下slaveId
        try {
            canalService.modify(canal);
        } catch (RepeatConfigureException rce) {
            err.setMessage("invalidCanal");
            return;
        }
    }
    nav.redirectTo(WebConstant.CANAL_LIST_LINK);
}

Example 7 with Canal

use of com.alibaba.otter.canal.instance.manager.model.Canal in project otter by alibaba.

the class CanalAction method doEdit.

/**
     * 修改canal
     */
public void doEdit(@FormGroup("canalInfo") Group canalInfo, @FormGroup("canalParameterInfo") Group canalParameterInfo, @FormField(name = "formCanalError", group = "canalInfo") CustomErrors err, @FormField(name = "formHeartBeatError", group = "canalParameterInfo") CustomErrors heartBeatErr, Navigator nav) throws Exception {
    Canal canal = new Canal();
    CanalParameter parameter = new CanalParameter();
    canalInfo.setProperties(canal);
    canalParameterInfo.setProperties(parameter);
    String zkClustersString = canalParameterInfo.getField("zkClusters").getStringValue();
    String[] zkClusters = StringUtils.split(zkClustersString, ";");
    parameter.setZkClusters(Arrays.asList(zkClusters));
    Long zkClusterId = canalParameterInfo.getField("autoKeeperClusterId").getLongValue();
    parameter.setZkClusterId(zkClusterId);
    String dbAddressesString = canalParameterInfo.getField("groupDbAddresses").getStringValue();
    if (StringUtils.isNotEmpty(dbAddressesString)) {
        List<List<DataSourcing>> dbSocketAddress = new ArrayList<List<DataSourcing>>();
        String[] dbAddresses = StringUtils.split(dbAddressesString, ";");
        for (String dbAddressString : dbAddresses) {
            List<DataSourcing> groupDbSocketAddress = new ArrayList<DataSourcing>();
            String[] groupDbAddresses = StringUtils.split(dbAddressString, ",");
            for (String groupDbAddress : groupDbAddresses) {
                String[] strs = StringUtils.split(groupDbAddress, ":");
                InetSocketAddress address = new InetSocketAddress(strs[0].trim(), Integer.valueOf(strs[1]));
                SourcingType type = parameter.getSourcingType();
                if (strs.length > 2) {
                    type = SourcingType.valueOf(strs[2]);
                }
                groupDbSocketAddress.add(new DataSourcing(type, address));
            }
            dbSocketAddress.add(groupDbSocketAddress);
        }
        parameter.setGroupDbAddresses(dbSocketAddress);
    }
    String positionsString = canalParameterInfo.getField("positions").getStringValue();
    if (StringUtils.isNotEmpty(positionsString)) {
        String[] positions = StringUtils.split(positionsString, ";");
        parameter.setPositions(Arrays.asList(positions));
    }
    if (parameter.getDetectingEnable() && StringUtils.startsWithIgnoreCase(parameter.getDetectingSQL(), "select")) {
        heartBeatErr.setMessage("invaliedHeartBeat");
        return;
    }
    canal.setCanalParameter(parameter);
    try {
        canalService.modify(canal);
    } catch (RepeatConfigureException rce) {
        err.setMessage("invalidCanal");
        return;
    }
    nav.redirectToLocation("canalList.htm");
}

Example 8 with Canal

use of com.alibaba.otter.canal.instance.manager.model.Canal in project otter by alibaba.

the class CanalServiceImpl method listByIds.

public List<Canal> listByIds(Long... identities) {
    List<Canal> canals = new ArrayList<Canal>();
    try {
        List<CanalDO> canalDos = null;
        if (identities.length < 1) {
            canalDos = canalDao.listAll();
            if (canalDos.isEmpty()) {
                logger.debug("DEBUG ## couldn't query any canal, maybe hasn't create any canal.");
                return canals;
            }
        } else {
            canalDos = canalDao.listByMultiId(identities);
            if (canalDos.isEmpty()) {
                String exceptionCause = "couldn't query any canal by canalIds:" + Arrays.toString(identities);
                logger.error("ERROR ## " + exceptionCause);
                throw new ManagerException(exceptionCause);
            }
        }
        canals = doToModel(canalDos);
    } catch (Exception e) {
        logger.error("ERROR ## query channels has an exception!");
        throw new ManagerException(e);
    }
    return canals;
}

Example 9 with Canal

use of com.alibaba.otter.canal.instance.manager.model.Canal in project otter by alibaba.

the class CanalEmbedSelector method start.

public void start() {
    if (running) {
        return;
    }
    // 获取destination/filter参数
    Pipeline pipeline = configClientService.findPipeline(pipelineId);
    filter = CanalFilterSupport.makeFilterExpression(pipeline);
    destination = pipeline.getParameters().getDestinationName();
    batchSize = pipeline.getParameters().getMainstemBatchsize();
    batchTimeout = pipeline.getParameters().getBatchTimeout();
    ddlSync = pipeline.getParameters().getDdlSync();
    final boolean syncFull = pipeline.getParameters().getSyncMode().isRow() || pipeline.getParameters().isEnableRemedy();
    // 暂时使用skip load代替
    filterTableError = pipeline.getParameters().getSkipSelectException();
    if (pipeline.getParameters().getDumpSelector() != null) {
        dump = pipeline.getParameters().getDumpSelector();
    }
    if (pipeline.getParameters().getDumpSelectorDetail() != null) {
        dumpDetail = pipeline.getParameters().getDumpSelectorDetail();
    }
    canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

        public CanalInstance generate(String destination) {
            Canal canal = canalConfigClient.findCanal(destination);
            final OtterAlarmHandler otterAlarmHandler = new OtterAlarmHandler();
            otterAlarmHandler.setPipelineId(pipelineId);
            // 注入一下spring资源
            OtterContextLocator.autowire(otterAlarmHandler);
            // 设置下slaveId，保证多个piplineId下重复引用时不重复
            // 默认基数
            long slaveId = 10000;
            if (canal.getCanalParameter().getSlaveId() != null) {
                slaveId = canal.getCanalParameter().getSlaveId();
            }
            canal.getCanalParameter().setSlaveId(slaveId + pipelineId);
            canal.getCanalParameter().setDdlIsolation(ddlSync);
            canal.getCanalParameter().setFilterTableError(filterTableError);
            CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, filter) {

                protected CanalHAController initHaController() {
                    HAMode haMode = parameters.getHaMode();
                    if (haMode.isMedia()) {
                        return new MediaHAController(parameters.getMediaGroup(), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName());
                    } else {
                        return super.initHaController();
                    }
                }

                protected void startEventParserInternal(CanalEventParser parser, boolean isGroup) {
                    super.startEventParserInternal(parser, isGroup);
                    if (eventParser instanceof MysqlEventParser) {
                        // 设置支持的类型
                        ((MysqlEventParser) eventParser).setSupportBinlogFormats("ROW");
                        if (syncFull) {
                            ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL");
                        } else {
                            ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL,MINIMAL");
                        }
                        MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
                        CanalHAController haController = mysqlEventParser.getHaController();
                        if (haController instanceof MediaHAController) {
                            if (isGroup) {
                                throw new CanalException("not support group database use media HA");
                            }
                            ((MediaHAController) haController).setCanalHASwitchable(mysqlEventParser);
                        }
                        if (!haController.isStart()) {
                            haController.start();
                        }
                        // 基于media的Ha，直接从tddl中获取数据库信息
                        if (haController instanceof MediaHAController) {
                            AuthenticationInfo authenticationInfo = ((MediaHAController) haController).getAvailableAuthenticationInfo();
                            ((MysqlEventParser) eventParser).setMasterInfo(authenticationInfo);
                        }
                    }
                }
            };
            instance.setAlarmHandler(otterAlarmHandler);
            CanalEventSink eventSink = instance.getEventSink();
            if (eventSink instanceof AbstractCanalEventSink) {
                handler = new OtterDownStreamHandler();
                handler.setPipelineId(pipelineId);
                handler.setDetectingIntervalInSeconds(canal.getCanalParameter().getDetectingIntervalInSeconds());
                // 注入一下spring资源
                OtterContextLocator.autowire(handler);
                // 添加到开头
                ((AbstractCanalEventSink) eventSink).addHandler(handler, 0);
                handler.start();
            }
            return instance;
        }
    });
    canalServer.start();
    canalServer.start(destination);
    this.clientIdentity = new ClientIdentity(destination, pipeline.getParameters().getMainstemClientId(), filter);
    // 发起一次订阅
    canalServer.subscribe(clientIdentity);
    running = true;
}

Example 10 with Canal

use of com.alibaba.otter.canal.instance.manager.model.Canal in project otter by alibaba.

the class CanalInfo method execute.

public void execute(@Param("canalId") Long canalId, Context context) throws Exception {
    Canal canal = canalService.findById(canalId);
    AutoKeeperCluster zkCluster = autoKeeperClusterService.findAutoKeeperClusterById(canal.getCanalParameter().getZkClusterId());
    List<Pipeline> pipelines = pipelineService.listByDestinationWithoutOther(canal.getName());
    List<Long> channelIds = new ArrayList<Long>();
    for (Pipeline pipeline : pipelines) {
        channelIds.add(pipeline.getChannelId());
    }
    List<Channel> channels = channelService.listOnlyChannels(channelIds.toArray(new Long[channelIds.size()]));
    Map<Long, Channel> channelMap = new HashMap<Long, Channel>();
    for (Channel channel : channels) {
        channelMap.put(channel.getId(), channel);
    }
    context.put("canal", canal);
    context.put("pipelines", pipelines);
    context.put("channelMap", channelMap);
    context.put("zkCluster", zkCluster);
}