Examples with EntryRowData com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData used on opensource projects

Example 1 with EntryRowData

use of com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData in project canal by alibaba.

the class CanalKafkaProducer method send.

private List<Future> send(MQDestination mqDestination, String topicName, Message message, boolean flat) {
    List<ProducerRecord<String, byte[]>> records = new ArrayList<>();
    // 获取当前topic的分区数
    Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, mqDestination.getDynamicTopicPartitionNum());
    if (partitionNum == null) {
        partitionNum = mqDestination.getPartitionsNum();
    }
    if (!flat) {
        if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
            // 并发构造
            EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
            // 串行分区
            Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), partitionNum, mqDestination.getPartitionHash(), this.mqProperties.isDatabaseHash());
            int length = messages.length;
            for (int i = 0; i < length; i++) {
                Message messagePartition = messages[i];
                if (messagePartition != null) {
                    records.add(new ProducerRecord<>(topicName, i, null, CanalMessageSerializerUtil.serializer(messagePartition, mqProperties.isFilterTransactionEntry())));
                }
            }
        } else {
            final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
            records.add(new ProducerRecord<>(topicName, partition, null, CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry())));
        }
    } else {
        // 发送扁平数据json
        // 并发构造
        EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
        for (FlatMessage flatMessage : flatMessages) {
            if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
                FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, partitionNum, mqDestination.getPartitionHash(), this.mqProperties.isDatabaseHash());
                int length = partitionFlatMessage.length;
                for (int i = 0; i < length; i++) {
                    FlatMessage flatMessagePart = partitionFlatMessage[i];
                    if (flatMessagePart != null) {
                        records.add(new ProducerRecord<>(topicName, i, null, JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue)));
                    }
                }
            } else {
                final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
                records.add(new ProducerRecord<>(topicName, partition, null, JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)));
            }
        }
    }
    return produce(records);
}