Examples with RocketMQProducerConfig com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig used on opensource projects

Example 1 with RocketMQProducerConfig

use of com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig in project canal by alibaba.

the class CanalRocketMQProducer method loadRocketMQProperties.

private void loadRocketMQProperties(Properties properties) {
    RocketMQProducerConfig rocketMQProperties = (RocketMQProducerConfig) this.mqProperties;
    // 兼容下<=1.1.4的mq配置
    doMoreCompatibleConvert("canal.mq.servers", "rocketmq.namesrv.addr", properties);
    doMoreCompatibleConvert("canal.mq.producerGroup", "rocketmq.producer.group", properties);
    doMoreCompatibleConvert("canal.mq.namespace", "rocketmq.namespace", properties);
    doMoreCompatibleConvert("canal.mq.retries", "rocketmq.retry.times.when.send.failed", properties);
    String producerGroup = PropertiesUtils.getProperty(properties, RocketMQConstants.ROCKETMQ_PRODUCER_GROUP);
    if (!StringUtils.isEmpty(producerGroup)) {
        rocketMQProperties.setProducerGroup(producerGroup);
    }
    String enableMessageTrace = PropertiesUtils.getProperty(properties, RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
    if (!StringUtils.isEmpty(enableMessageTrace)) {
        rocketMQProperties.setEnableMessageTrace(Boolean.parseBoolean(enableMessageTrace));
    }
    String customizedTraceTopic = PropertiesUtils.getProperty(properties, RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
    if (!StringUtils.isEmpty(customizedTraceTopic)) {
        rocketMQProperties.setCustomizedTraceTopic(customizedTraceTopic);
    }
    String namespace = PropertiesUtils.getProperty(properties, RocketMQConstants.ROCKETMQ_NAMESPACE);
    if (!StringUtils.isEmpty(namespace)) {
        rocketMQProperties.setNamespace(namespace);
    }
    String namesrvAddr = PropertiesUtils.getProperty(properties, RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
    if (!StringUtils.isEmpty(namesrvAddr)) {
        rocketMQProperties.setNamesrvAddr(namesrvAddr);
    }
    String retry = PropertiesUtils.getProperty(properties, RocketMQConstants.ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED);
    if (!StringUtils.isEmpty(retry)) {
        rocketMQProperties.setRetryTimesWhenSendFailed(Integer.parseInt(retry));
    }
    String vipChannelEnabled = PropertiesUtils.getProperty(properties, RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED);
    if (!StringUtils.isEmpty(vipChannelEnabled)) {
        rocketMQProperties.setVipChannelEnabled(Boolean.parseBoolean(vipChannelEnabled));
    }
    String tag = PropertiesUtils.getProperty(properties, RocketMQConstants.ROCKETMQ_TAG);
    if (!StringUtils.isEmpty(tag)) {
        rocketMQProperties.setTag(tag);
    }
}

Example 2 with RocketMQProducerConfig

use of com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig in project canal by alibaba.

the class CanalRocketMQProducer method send.

public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
    // 获取当前topic的分区数
    Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, destination.getDynamicTopicPartitionNum());
    // 获取topic的队列数为分区数
    if (partitionNum == null) {
        partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
    }
    if (partitionNum == null) {
        partitionNum = destination.getPartitionsNum();
    }
    if (!mqProperties.isFlatMessage()) {
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            // 并发构造
            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
            // 串行分区
            com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
            int length = messages.length;
            ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
            for (int i = 0; i < length; i++) {
                com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
                if (dataPartition != null) {
                    final int index = i;
                    template.submit(() -> {
                        Message data = new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), CanalMessageSerializerUtil.serializer(dataPartition, mqProperties.isFilterTransactionEntry()));
                        sendMessage(data, index);
                    });
                }
            }
            // 等所有分片发送完毕
            template.waitForResult();
        } else {
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            Message data = new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
            sendMessage(data, partition);
        }
    } else {
        // 并发构造
        MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
        // 初始化分区合并队列
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
            for (int i = 0; i < partitionNum; i++) {
                partitionFlatMessages.add(new ArrayList<>());
            }
            for (FlatMessage flatMessage : flatMessages) {
                FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
                int length = partitionFlatMessage.length;
                for (int i = 0; i < length; i++) {
                    // 增加null判断,issue #3267
                    if (partitionFlatMessage[i] != null) {
                        partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
                    }
                }
            }
            ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
            for (int i = 0; i < partitionFlatMessages.size(); i++) {
                final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
                if (flatMessagePart != null && flatMessagePart.size() > 0) {
                    final int index = i;
                    template.submit(() -> {
                        List<Message> messages = flatMessagePart.stream().map(flatMessage -> new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))).collect(Collectors.toList());
                        // 批量发送
                        sendMessage(messages, index);
                    });
                }
            }
            // 批量等所有分区的结果
            template.waitForResult();
        } else {
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            List<Message> messages = flatMessages.stream().map(flatMessage -> new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))).collect(Collectors.toList());
            // 批量发送
            sendMessage(messages, partition);
        }
    }
}

Example 3 with RocketMQProducerConfig

use of com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig in project canal by alibaba.

the class CanalRocketMQProducer method init.

@Override
public void init(Properties properties) {
    RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
    this.mqProperties = rocketMQProperties;
    super.init(properties);
    loadRocketMQProperties(properties);
    RPCHook rpcHook = null;
    if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0) {
        SessionCredentials sessionCredentials = new SessionCredentials();
        sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey());
        sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey());
        rpcHook = new AclClientRPCHook(sessionCredentials);
    }
    defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(), rpcHook, rocketMQProperties.isEnableMessageTrace(), rocketMQProperties.getCustomizedTraceTopic());
    if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
        defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
    }
    if (!StringUtils.isEmpty(rocketMQProperties.getNamespace())) {
        defaultMQProducer.setNamespace(rocketMQProperties.getNamespace());
    }
    defaultMQProducer.setNamesrvAddr(rocketMQProperties.getNamesrvAddr());
    defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetryTimesWhenSendFailed());
    defaultMQProducer.setVipChannelEnabled(rocketMQProperties.isVipChannelEnabled());
    logger.info("##Start RocketMQ producer##");
    try {
        defaultMQProducer.start();
    } catch (MQClientException ex) {
        throw new CanalException("Start RocketMQ producer error", ex);
    }
    int parallelPartitionSendThreadSize = mqProperties.getParallelSendThreadSize();
    sendPartitionExecutor = new ThreadPoolExecutor(parallelPartitionSendThreadSize, parallelPartitionSendThreadSize, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<>(parallelPartitionSendThreadSize * 2), new NamedThreadFactory("MQ-Parallel-Sender-Partition"), new ThreadPoolExecutor.CallerRunsPolicy());
}