Examples with MQDestination com.alibaba.otter.canal.connector.core.producer.MQDestination used on opensource projects

Example 1 with MQDestination

use of com.alibaba.otter.canal.connector.core.producer.MQDestination in project canal by alibaba.

the class CanalRocketMQProducer method send.

public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
    // 获取当前topic的分区数
    Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, destination.getDynamicTopicPartitionNum());
    // 获取topic的队列数为分区数
    if (partitionNum == null) {
        partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
    }
    if (partitionNum == null) {
        partitionNum = destination.getPartitionsNum();
    }
    if (!mqProperties.isFlatMessage()) {
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            // 并发构造
            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
            // 串行分区
            com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
            int length = messages.length;
            ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
            for (int i = 0; i < length; i++) {
                com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
                if (dataPartition != null) {
                    final int index = i;
                    template.submit(() -> {
                        Message data = new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), CanalMessageSerializerUtil.serializer(dataPartition, mqProperties.isFilterTransactionEntry()));
                        sendMessage(data, index);
                    });
                }
            }
            // 等所有分片发送完毕
            template.waitForResult();
        } else {
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            Message data = new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
            sendMessage(data, partition);
        }
    } else {
        // 并发构造
        MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
        // 初始化分区合并队列
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
            for (int i = 0; i < partitionNum; i++) {
                partitionFlatMessages.add(new ArrayList<>());
            }
            for (FlatMessage flatMessage : flatMessages) {
                FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
                int length = partitionFlatMessage.length;
                for (int i = 0; i < length; i++) {
                    // 增加null判断,issue #3267
                    if (partitionFlatMessage[i] != null) {
                        partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
                    }
                }
            }
            ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
            for (int i = 0; i < partitionFlatMessages.size(); i++) {
                final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
                if (flatMessagePart != null && flatMessagePart.size() > 0) {
                    final int index = i;
                    template.submit(() -> {
                        List<Message> messages = flatMessagePart.stream().map(flatMessage -> new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))).collect(Collectors.toList());
                        // 批量发送
                        sendMessage(messages, index);
                    });
                }
            }
            // 批量等所有分区的结果
            template.waitForResult();
        } else {
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            List<Message> messages = flatMessages.stream().map(flatMessage -> new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))).collect(Collectors.toList());
            // 批量发送
            sendMessage(messages, partition);
        }
    }
}

Example 2 with MQDestination

use of com.alibaba.otter.canal.connector.core.producer.MQDestination in project canal by alibaba.

the class CanalMQStarter method worker.

private void worker(String destination, AtomicBoolean destinationRunning) {
    while (!running || !destinationRunning.get()) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
        // ignore
        }
    }
    logger.info("## start the MQ producer: {}.", destination);
    MDC.put("destination", destination);
    final ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1001, "");
    while (running && destinationRunning.get()) {
        try {
            CanalInstance canalInstance = canalServer.getCanalInstances().get(destination);
            if (canalInstance == null) {
                try {
                    Thread.sleep(3000);
                } catch (InterruptedException e) {
                // ignore
                }
                continue;
            }
            MQDestination canalDestination = new MQDestination();
            canalDestination.setCanalDestination(destination);
            CanalMQConfig mqConfig = canalInstance.getMqConfig();
            canalDestination.setTopic(mqConfig.getTopic());
            canalDestination.setPartition(mqConfig.getPartition());
            canalDestination.setDynamicTopic(mqConfig.getDynamicTopic());
            canalDestination.setPartitionsNum(mqConfig.getPartitionsNum());
            canalDestination.setPartitionHash(mqConfig.getPartitionHash());
            canalDestination.setDynamicTopicPartitionNum(mqConfig.getDynamicTopicPartitionNum());
            canalDestination.setEnableDynamicQueuePartition(mqConfig.getEnableDynamicQueuePartition());
            canalServer.subscribe(clientIdentity);
            logger.info("## the MQ producer: {} is running now ......", destination);
            Integer getTimeout = mqProperties.getFetchTimeout();
            Integer getBatchSize = mqProperties.getBatchSize();
            while (running && destinationRunning.get()) {
                Message message;
                if (getTimeout != null && getTimeout > 0) {
                    message = canalServer.getWithoutAck(clientIdentity, getBatchSize, getTimeout.longValue(), TimeUnit.MILLISECONDS);
                } else {
                    message = canalServer.getWithoutAck(clientIdentity, getBatchSize);
                }
                final long batchId = message.getId();
                try {
                    int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
                    if (batchId != -1 && size != 0) {
                        canalMQProducer.send(canalDestination, message, new Callback() {

                            @Override
                            public void commit() {
                                // 提交确认
                                canalServer.ack(clientIdentity, batchId);
                            }

                            @Override
                            public void rollback() {
                                canalServer.rollback(clientIdentity, batchId);
                            }
                        });
                    // 发送message到topic
                    } else {
                        try {
                            Thread.sleep(100);
                        } catch (InterruptedException e) {
                        // ignore
                        }
                    }
                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                }
            }
        } catch (Exception e) {
            logger.error("process error!", e);
        }
    }
}