Examples with ExecutorTemplate com.alibaba.otter.canal.common.utils.ExecutorTemplate used on opensource projects

Example 1 with ExecutorTemplate

use of com.alibaba.otter.canal.common.utils.ExecutorTemplate in project canal by alibaba.

the class CanalPulsarMQProducer method send.

/**
 * 发送单条消息到指定topic。区分是否发送扁平消息
 *
 * @param destination
 * @param topicName
 * @param message
 * @return void
 * @date 2021/9/2 22:05
 * @author chad
 * @since 1.0.0 by chad at 2021/9/2: 新增
 */
public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
    // 获取当前topic的分区数
    Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, destination.getDynamicTopicPartitionNum());
    if (partitionNum == null) {
        partitionNum = destination.getPartitionsNum();
    }
    ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
    // 并发构造
    MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
    if (!mqProperties.isFlatMessage()) {
        // 动态计算目标分区
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            for (MQMessageUtils.EntryRowData r : datas) {
                CanalEntry.Entry entry = r.entry;
                if (null == entry) {
                    continue;
                }
                // 串行分区
                com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
                // 发送
                int len = messages.length;
                for (int i = 0; i < len; i++) {
                    final int partition = i;
                    com.alibaba.otter.canal.protocol.Message m = messages[i];
                    template.submit(() -> {
                        sendMessage(topicName, partition, m);
                    });
                }
            }
        } else {
            // 默认分区
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            sendMessage(topicName, partition, message);
        }
    } else {
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
        // 初始化分区合并队列
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
            int len = partitionNum;
            for (int i = 0; i < len; i++) {
                partitionFlatMessages.add(new ArrayList<>());
            }
            for (FlatMessage flatMessage : flatMessages) {
                FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
                int length = partitionFlatMessage.length;
                for (int i = 0; i < length; i++) {
                    // 增加null判断,issue #3267
                    if (partitionFlatMessage[i] != null) {
                        partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
                    }
                }
            }
            for (int i = 0; i < len; i++) {
                final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
                if (flatMessagePart != null && flatMessagePart.size() > 0) {
                    final int partition = i;
                    template.submit(() -> {
                        // 批量发送
                        sendMessage(topicName, partition, flatMessagePart);
                    });
                }
            }
            // 批量等所有分区的结果
            template.waitForResult();
        } else {
            // 默认分区
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            sendMessage(topicName, partition, flatMessages);
        }
    }
}

Example 2 with ExecutorTemplate

use of com.alibaba.otter.canal.common.utils.ExecutorTemplate in project canal by alibaba.

the class CanalKafkaProducer method send.

@Override
public void send(MQDestination mqDestination, Message message, Callback callback) {
    ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
    try {
        List result;
        if (!StringUtils.isEmpty(mqDestination.getDynamicTopic())) {
            // 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化
            Map<String, Message> messageMap = MQMessageUtils.messageTopics(message, mqDestination.getTopic(), mqDestination.getDynamicTopic());
            // 针对不同的topic,引入多线程提升效率
            for (Map.Entry<String, Message> entry : messageMap.entrySet()) {
                final String topicName = entry.getKey().replace('.', '_');
                final Message messageSub = entry.getValue();
                template.submit((Callable) () -> {
                    try {
                        return send(mqDestination, topicName, messageSub, mqProperties.isFlatMessage());
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                });
            }
            result = template.waitForResult();
        } else {
            result = new ArrayList();
            List<Future> futures = send(mqDestination, mqDestination.getTopic(), message, mqProperties.isFlatMessage());
            result.add(futures);
        }
        // 一个批次的所有topic和分区的队列，都采用异步的模式进行多线程批量发送
        // 最后在集结点进行flush等待，确保所有数据都写出成功
        // 注意：kafka的异步模式如果要保证顺序性，需要设置max.in.flight.requests.per.connection=1，确保在网络异常重试时有排他性
        producer.flush();
        // flush操作也有可能是发送失败,这里需要异步关注一下发送结果,针对有异常的直接出发rollback
        for (Object obj : result) {
            List<Future> futures = (List<Future>) obj;
            for (Future future : futures) {
                try {
                    future.get();
                } catch (InterruptedException | ExecutionException e) {
                    throw new RuntimeException(e);
                }
            }
        }
        callback.commit();
    } catch (Throwable e) {
        logger.error(e.getMessage(), e);
        callback.rollback();
    } finally {
        template.clear();
    }
}

Example 3 with ExecutorTemplate

use of com.alibaba.otter.canal.common.utils.ExecutorTemplate in project canal by alibaba.

the class CanalRocketMQProducer method send.

public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
    // 获取当前topic的分区数
    Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, destination.getDynamicTopicPartitionNum());
    // 获取topic的队列数为分区数
    if (partitionNum == null) {
        partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
    }
    if (partitionNum == null) {
        partitionNum = destination.getPartitionsNum();
    }
    if (!mqProperties.isFlatMessage()) {
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            // 并发构造
            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
            // 串行分区
            com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
            int length = messages.length;
            ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
            for (int i = 0; i < length; i++) {
                com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
                if (dataPartition != null) {
                    final int index = i;
                    template.submit(() -> {
                        Message data = new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), CanalMessageSerializerUtil.serializer(dataPartition, mqProperties.isFilterTransactionEntry()));
                        sendMessage(data, index);
                    });
                }
            }
            // 等所有分片发送完毕
            template.waitForResult();
        } else {
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            Message data = new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
            sendMessage(data, partition);
        }
    } else {
        // 并发构造
        MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
        // 初始化分区合并队列
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
            for (int i = 0; i < partitionNum; i++) {
                partitionFlatMessages.add(new ArrayList<>());
            }
            for (FlatMessage flatMessage : flatMessages) {
                FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
                int length = partitionFlatMessage.length;
                for (int i = 0; i < length; i++) {
                    // 增加null判断,issue #3267
                    if (partitionFlatMessage[i] != null) {
                        partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
                    }
                }
            }
            ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
            for (int i = 0; i < partitionFlatMessages.size(); i++) {
                final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
                if (flatMessagePart != null && flatMessagePart.size() > 0) {
                    final int index = i;
                    template.submit(() -> {
                        List<Message> messages = flatMessagePart.stream().map(flatMessage -> new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))).collect(Collectors.toList());
                        // 批量发送
                        sendMessage(messages, index);
                    });
                }
            }
            // 批量等所有分区的结果
            template.waitForResult();
        } else {
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            List<Message> messages = flatMessages.stream().map(flatMessage -> new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))).collect(Collectors.toList());
            // 批量发送
            sendMessage(messages, partition);
        }
    }
}

Example 4 with ExecutorTemplate

use of com.alibaba.otter.canal.common.utils.ExecutorTemplate in project canal by alibaba.

the class CanalRocketMQProducer method send.

@Override
public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
    ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
    try {
        if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
            // 动态topic
            Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
            for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
                String topicName = entry.getKey().replace('.', '_');
                com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
                template.submit(() -> {
                    try {
                        send(destination, topicName, messageSub);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                });
            }
            template.waitForResult();
        } else {
            send(destination, destination.getTopic(), message);
        }
        callback.commit();
    } catch (Throwable e) {
        logger.error(e.getMessage(), e);
        callback.rollback();
    } finally {
        template.clear();
    }
}

Example 5 with ExecutorTemplate

use of com.alibaba.otter.canal.common.utils.ExecutorTemplate in project canal by alibaba.

the class MQMessageUtils method buildMessageData.

/**
 * 多线程构造message的rowChanged对象，比如为partition/flastMessage转化等处理 </br>
 * 因为protobuf对象的序列化和反序列化是cpu密集型，串行执行会有代价
 */
public static EntryRowData[] buildMessageData(Message message, ThreadPoolExecutor executor) {
    ExecutorTemplate template = new ExecutorTemplate(executor);
    if (message.isRaw()) {
        List<ByteString> rawEntries = message.getRawEntries();
        final EntryRowData[] datas = new EntryRowData[rawEntries.size()];
        int i = 0;
        for (ByteString byteString : rawEntries) {
            final int index = i;
            template.submit(() -> {
                try {
                    Entry entry = Entry.parseFrom(byteString);
                    RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
                    datas[index] = new EntryRowData();
                    datas[index].entry = entry;
                    datas[index].rowChange = rowChange;
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException(e);
                }
            });
            i++;
        }
        template.waitForResult();
        return datas;
    } else {
        final EntryRowData[] datas = new EntryRowData[message.getEntries().size()];
        int i = 0;
        for (Entry entry : message.getEntries()) {
            final int index = i;
            template.submit(() -> {
                try {
                    RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
                    datas[index] = new EntryRowData();
                    datas[index].entry = entry;
                    datas[index].rowChange = rowChange;
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException(e);
                }
            });
            i++;
        }
        template.waitForResult();
        return datas;
    }
}