Examples with Message com.alibaba.otter.canal.protocol.Message used on opensource projects

Example 11 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class PulsarMQCanalConnector method getListWithoutAck.

/**
 * 获取泛型数据，供其他方法调用
 * <p>
 * 不支持多线程调用
 * </p>
 *
 * @return java.util.List<T>
 * @date 2021/9/14 15:20
 * @author chad
 * @since 1 by chad at 2021/9/14 供{@link PulsarMQCanalConnector#getListWithoutAck(Long, TimeUnit)}
 * 和{@link PulsarMQCanalConnector#getFlatListWithoutAck(Long, TimeUnit)}调用
 */
private <T> List<T> getListWithoutAck() {
    if (null != this.lastGetBatchMessage) {
        throw new CanalClientException("mq get/ack not support concurrent & async ack");
    }
    List messageList = Lists.newArrayList();
    try {
        this.lastGetBatchMessage = consumer.batchReceive();
        if (null == this.lastGetBatchMessage || this.lastGetBatchMessage.size() < 1) {
            this.lastGetBatchMessage = null;
            return messageList;
        }
    } catch (PulsarClientException e) {
        logger.error("Receiver Pulsar MQ message error", e);
        throw new CanalClientException(e);
    }
    for (org.apache.pulsar.client.api.Message<byte[]> msgExt : this.lastGetBatchMessage) {
        byte[] data = msgExt.getData();
        if (data == null) {
            logger.warn("Received message data is null");
            continue;
        }
        try {
            if (isFlatMessage) {
                FlatMessage flatMessage = JSON.parseObject(data, FlatMessage.class);
                messageList.add(flatMessage);
            } else {
                Message message = CanalMessageDeserializer.deserializer(data);
                messageList.add(message);
            }
        } catch (Exception ex) {
            logger.error("Add message error", ex);
            throw new CanalClientException(ex);
        }
    }
    return messageList;
}

Example 12 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class MQMessageUtils method messageTopics.

/**
 * 按 schema 或者 schema+table 将 message 分配到对应topic
 *
 * @param message 原message
 * @param defaultTopic 默认topic
 * @param dynamicTopicConfigs 动态topic规则
 * @return 分隔后的message map
 */
public static Map<String, Message> messageTopics(Message message, String defaultTopic, String dynamicTopicConfigs) {
    List<CanalEntry.Entry> entries;
    if (message.isRaw()) {
        List<ByteString> rawEntries = message.getRawEntries();
        entries = new ArrayList<>(rawEntries.size());
        for (ByteString byteString : rawEntries) {
            CanalEntry.Entry entry;
            try {
                entry = CanalEntry.Entry.parseFrom(byteString);
            } catch (InvalidProtocolBufferException e) {
                throw new RuntimeException(e);
            }
            entries.add(entry);
        }
    } else {
        entries = message.getEntries();
    }
    Map<String, Message> messages = new HashMap<>();
    for (CanalEntry.Entry entry : entries) {
        // 如果有topic路由,则忽略begin/end事件
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
            continue;
        }
        String schemaName = entry.getHeader().getSchemaName();
        String tableName = entry.getHeader().getTableName();
        if (StringUtils.isEmpty(schemaName) || StringUtils.isEmpty(tableName)) {
            put2MapMessage(messages, message.getId(), defaultTopic, entry);
        } else {
            Set<String> topics = matchTopics(schemaName + "." + tableName, dynamicTopicConfigs);
            if (topics != null) {
                for (String topic : topics) {
                    put2MapMessage(messages, message.getId(), topic, entry);
                }
            } else {
                topics = matchTopics(schemaName, dynamicTopicConfigs);
                if (topics != null) {
                    for (String topic : topics) {
                        put2MapMessage(messages, message.getId(), topic, entry);
                    }
                } else {
                    put2MapMessage(messages, message.getId(), defaultTopic, entry);
                }
            }
        }
    }
    return messages;
}

Example 13 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class SimpleCanalConnector method get.

@Override
public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
    Message message = getWithoutAck(batchSize, timeout, unit);
    ack(message.getId());
    return message;
}

Example 14 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class RocketMQCanalConnector method process.

private boolean process(List<MessageExt> messageExts) {
    if (logger.isDebugEnabled()) {
        logger.debug("Get Message: {}", messageExts);
    }
    List messageList = new ArrayList<>();
    for (MessageExt messageExt : messageExts) {
        byte[] data = messageExt.getBody();
        if (data != null) {
            try {
                if (!flatMessage) {
                    Message message = CanalMessageDeserializer.deserializer(data);
                    messageList.add(message);
                } else {
                    FlatMessage flatMessage = JSON.parseObject(data, FlatMessage.class);
                    messageList.add(flatMessage);
                }
            } catch (Exception ex) {
                logger.error("Add message error", ex);
                throw new CanalClientException(ex);
            }
        } else {
            logger.warn("Received message data is null");
        }
    }
    ConsumerBatchMessage batchMessage;
    if (!flatMessage) {
        batchMessage = new ConsumerBatchMessage<Message>(messageList);
    } else {
        batchMessage = new ConsumerBatchMessage<FlatMessage>(messageList);
    }
    try {
        messageBlockingQueue.put(batchMessage);
    } catch (InterruptedException e) {
        logger.error("Put message to queue error", e);
        throw new RuntimeException(e);
    }
    boolean isCompleted;
    try {
        isCompleted = batchMessage.waitFinish(batchProcessTimeout);
    } catch (InterruptedException e) {
        logger.error("Interrupted when waiting messages to be finished.", e);
        throw new RuntimeException(e);
    }
    boolean isSuccess = batchMessage.isSuccess();
    return isCompleted && isSuccess;
}

Example 15 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class CanalRocketMQConsumer method process.

private boolean process(List<MessageExt> messageExts) {
    if (logger.isDebugEnabled()) {
        logger.debug("Get Message: {}", messageExts);
    }
    List<CommonMessage> messageList = new ArrayList<>();
    for (MessageExt messageExt : messageExts) {
        byte[] data = messageExt.getBody();
        if (data != null) {
            try {
                if (!flatMessage) {
                    Message message = CanalMessageSerializerUtil.deserializer(data);
                    messageList.addAll(MessageUtil.convert(message));
                } else {
                    CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
                    messageList.add(commonMessage);
                }
            } catch (Exception ex) {
                logger.error("Add message error", ex);
                throw new CanalClientException(ex);
            }
        } else {
            logger.warn("Received message data is null");
        }
    }
    ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
    try {
        messageBlockingQueue.put(batchMessage);
    } catch (InterruptedException e) {
        logger.error("Put message to queue error", e);
        throw new RuntimeException(e);
    }
    boolean isCompleted;
    try {
        isCompleted = batchMessage.waitFinish(batchProcessTimeout);
    } catch (InterruptedException e) {
        logger.error("Interrupted when waiting messages to be finished.", e);
        throw new RuntimeException(e);
    }
    boolean isSuccess = batchMessage.isSuccess();
    return isCompleted && isSuccess;
}