Examples with FlatMessage com.alibaba.otter.canal.protocol.FlatMessage used on opensource projects

Example 1 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class CanalRocketMQClientFlatMessageExample method process.

private void process() {
    while (!running) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        }
    }
    while (running) {
        try {
            connector.connect();
            connector.subscribe();
            while (running) {
                // 获取message
                List<FlatMessage> messages = connector.getFlatList(100L, TimeUnit.MILLISECONDS);
                for (FlatMessage message : messages) {
                    long batchId = message.getId();
                    if (batchId == -1 || message.getData() == null) {
                    // try {
                    // Thread.sleep(1000);
                    // } catch (InterruptedException e) {
                    // }
                    } else {
                        logger.info(message.toString());
                    }
                }
                // 提交确认
                connector.ack();
            }
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
        }
    }
    connector.unsubscribe();
// connector.stopRunning();
}

Example 2 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class CanalPulsarMQProducer method send.

/**
 * 发送单条消息到指定topic。区分是否发送扁平消息
 *
 * @param destination
 * @param topicName
 * @param message
 * @return void
 * @date 2021/9/2 22:05
 * @author chad
 * @since 1.0.0 by chad at 2021/9/2: 新增
 */
public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
    // 获取当前topic的分区数
    Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, destination.getDynamicTopicPartitionNum());
    if (partitionNum == null) {
        partitionNum = destination.getPartitionsNum();
    }
    ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
    // 并发构造
    MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
    if (!mqProperties.isFlatMessage()) {
        // 动态计算目标分区
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            for (MQMessageUtils.EntryRowData r : datas) {
                CanalEntry.Entry entry = r.entry;
                if (null == entry) {
                    continue;
                }
                // 串行分区
                com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
                // 发送
                int len = messages.length;
                for (int i = 0; i < len; i++) {
                    final int partition = i;
                    com.alibaba.otter.canal.protocol.Message m = messages[i];
                    template.submit(() -> {
                        sendMessage(topicName, partition, m);
                    });
                }
            }
        } else {
            // 默认分区
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            sendMessage(topicName, partition, message);
        }
    } else {
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
        // 初始化分区合并队列
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
            int len = partitionNum;
            for (int i = 0; i < len; i++) {
                partitionFlatMessages.add(new ArrayList<>());
            }
            for (FlatMessage flatMessage : flatMessages) {
                FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
                int length = partitionFlatMessage.length;
                for (int i = 0; i < length; i++) {
                    // 增加null判断,issue #3267
                    if (partitionFlatMessage[i] != null) {
                        partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
                    }
                }
            }
            for (int i = 0; i < len; i++) {
                final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
                if (flatMessagePart != null && flatMessagePart.size() > 0) {
                    final int partition = i;
                    template.submit(() -> {
                        // 批量发送
                        sendMessage(topicName, partition, flatMessagePart);
                    });
                }
            }
            // 批量等所有分区的结果
            template.waitForResult();
        } else {
            // 默认分区
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            sendMessage(topicName, partition, flatMessages);
        }
    }
}

Example 3 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class CanalKafkaProducer method send.

private List<Future> send(MQDestination mqDestination, String topicName, Message message, boolean flat) {
    List<ProducerRecord<String, byte[]>> records = new ArrayList<>();
    // 获取当前topic的分区数
    Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, mqDestination.getDynamicTopicPartitionNum());
    if (partitionNum == null) {
        partitionNum = mqDestination.getPartitionsNum();
    }
    if (!flat) {
        if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
            // 并发构造
            EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
            // 串行分区
            Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), partitionNum, mqDestination.getPartitionHash(), this.mqProperties.isDatabaseHash());
            int length = messages.length;
            for (int i = 0; i < length; i++) {
                Message messagePartition = messages[i];
                if (messagePartition != null) {
                    records.add(new ProducerRecord<>(topicName, i, null, CanalMessageSerializerUtil.serializer(messagePartition, mqProperties.isFilterTransactionEntry())));
                }
            }
        } else {
            final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
            records.add(new ProducerRecord<>(topicName, partition, null, CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry())));
        }
    } else {
        // 发送扁平数据json
        // 并发构造
        EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
        for (FlatMessage flatMessage : flatMessages) {
            if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
                FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, partitionNum, mqDestination.getPartitionHash(), this.mqProperties.isDatabaseHash());
                int length = partitionFlatMessage.length;
                for (int i = 0; i < length; i++) {
                    FlatMessage flatMessagePart = partitionFlatMessage[i];
                    if (flatMessagePart != null) {
                        records.add(new ProducerRecord<>(topicName, i, null, JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue)));
                    }
                }
            } else {
                final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
                records.add(new ProducerRecord<>(topicName, partition, null, JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)));
            }
        }
    }
    return produce(records);
}

Example 4 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class KafkaOffsetCanalConnector method getFlatListWithoutAck.

/**
 * 获取Kafka消息，不确认
 *
 * @param timeout
 * @param unit
 * @param offset  消息偏移地址（-1为不偏移）
 * @return
 * @throws CanalClientException
 */
public List<KafkaFlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return new ArrayList<>();
    }
    if (offset > -1) {
        TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
        kafkaConsumer2.seek(tp, offset);
    }
    ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
    if (!records.isEmpty()) {
        List<KafkaFlatMessage> flatMessages = new ArrayList<>();
        for (ConsumerRecord<String, String> record : records) {
            String flatMessageJson = record.value();
            FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
            KafkaFlatMessage message = new KafkaFlatMessage(flatMessage, record.offset());
            flatMessages.add(message);
        }
        return flatMessages;
    }
    return new ArrayList<>();
}

Example 5 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class PulsarMQCanalConnector method getListWithoutAck.

/**
 * 获取泛型数据，供其他方法调用
 * <p>
 * 不支持多线程调用
 * </p>
 *
 * @return java.util.List<T>
 * @date 2021/9/14 15:20
 * @author chad
 * @since 1 by chad at 2021/9/14 供{@link PulsarMQCanalConnector#getListWithoutAck(Long, TimeUnit)}
 * 和{@link PulsarMQCanalConnector#getFlatListWithoutAck(Long, TimeUnit)}调用
 */
private <T> List<T> getListWithoutAck() {
    if (null != this.lastGetBatchMessage) {
        throw new CanalClientException("mq get/ack not support concurrent & async ack");
    }
    List messageList = Lists.newArrayList();
    try {
        this.lastGetBatchMessage = consumer.batchReceive();
        if (null == this.lastGetBatchMessage || this.lastGetBatchMessage.size() < 1) {
            this.lastGetBatchMessage = null;
            return messageList;
        }
    } catch (PulsarClientException e) {
        logger.error("Receiver Pulsar MQ message error", e);
        throw new CanalClientException(e);
    }
    for (org.apache.pulsar.client.api.Message<byte[]> msgExt : this.lastGetBatchMessage) {
        byte[] data = msgExt.getData();
        if (data == null) {
            logger.warn("Received message data is null");
            continue;
        }
        try {
            if (isFlatMessage) {
                FlatMessage flatMessage = JSON.parseObject(data, FlatMessage.class);
                messageList.add(flatMessage);
            } else {
                Message message = CanalMessageDeserializer.deserializer(data);
                messageList.add(message);
            }
        } catch (Exception ex) {
            logger.error("Add message error", ex);
            throw new CanalClientException(ex);
        }
    }
    return messageList;
}