Examples with Message com.alibaba.otter.canal.protocol.Message used on opensource projects

Example 26 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class CanalPulsarMQConsumer method getMessage.

@SuppressWarnings("unchecked")
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
    List<CommonMessage> messageList = Lists.newArrayList();
    try {
        Messages<byte[]> messages = pulsarMQConsumer.batchReceive();
        if (null == messages || messages.size() == 0) {
            return messageList;
        }
        // 保存当前消费记录，用于ack和rollback
        this.lastGetBatchMessage = messages;
        for (org.apache.pulsar.client.api.Message<byte[]> msg : messages) {
            byte[] data = msg.getData();
            if (!this.flatMessage) {
                Message message = CanalMessageSerializerUtil.deserializer(data);
                List<CommonMessage> list = MessageUtil.convert(message);
                messageList.addAll(list);
            } else {
                CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
                messageList.add(commonMessage);
            }
        }
    } catch (PulsarClientException e) {
        throw new CanalClientException("Receive pulsar batch message error", e);
    }
    return messageList;
}

Example 27 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class CanalRabbitMQProducer method send.

@Override
public void send(final MQDestination destination, Message message, Callback callback) {
    ExecutorTemplate template = new ExecutorTemplate(sendExecutor);
    try {
        if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
            // 动态topic
            Map<String, Message> messageMap = MQMessageUtils.messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
            for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
                final String topicName = entry.getKey().replace('.', '_');
                final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
                template.submit(() -> send(destination, topicName, messageSub));
            }
            template.waitForResult();
        } else {
            send(destination, destination.getTopic(), message);
        }
        callback.commit();
    } catch (Throwable e) {
        logger.error(e.getMessage(), e);
        callback.rollback();
    } finally {
        template.clear();
    }
}

Example 28 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class CanalRabbitMQConsumer method process.

private boolean process(byte[] messageData) {
    if (logger.isDebugEnabled()) {
        logger.debug("Get Message: {}", new String(messageData));
    }
    List<CommonMessage> messageList = new ArrayList<>();
    if (!flatMessage) {
        Message message = CanalMessageSerializerUtil.deserializer(messageData);
        messageList.addAll(MessageUtil.convert(message));
    } else {
        CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class);
        messageList.add(commonMessage);
    }
    ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
    try {
        messageBlockingQueue.put(batchMessage);
    } catch (InterruptedException e) {
        logger.error("Put message to queue error", e);
        throw new RuntimeException(e);
    }
    boolean isCompleted;
    try {
        isCompleted = batchMessage.waitFinish(batchProcessTimeout);
    } catch (InterruptedException e) {
        logger.error("Interrupted when waiting messages to be finished.", e);
        throw new RuntimeException(e);
    }
    boolean isSuccess = batchMessage.isSuccess();
    return isCompleted && isSuccess;
}

Example 29 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class KafkaCanalConnector method getListWithoutAck.

@Override
public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }
    ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
    currentOffsets.clear();
    for (TopicPartition topicPartition : records.partitions()) {
        currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition));
    }
    if (!records.isEmpty()) {
        List<Message> messages = new ArrayList<>();
        for (ConsumerRecord<String, Message> record : records) {
            messages.add(record.value());
        }
        return messages;
    }
    return Lists.newArrayList();
}

Example 30 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class KafkaOffsetCanalConnector method getListWithoutAck.

/**
 * 获取Kafka消息，不确认
 *
 * @param timeout
 * @param unit
 * @param offset  消息偏移地址（-1为不偏移）
 * @return
 * @throws CanalClientException
 */
public List<KafkaMessage> getListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return new ArrayList<>();
    }
    if (offset > -1) {
        TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
        kafkaConsumer.seek(tp, offset);
    }
    ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
    if (!records.isEmpty()) {
        List<KafkaMessage> messages = new ArrayList<>();
        for (ConsumerRecord<String, Message> record : records) {
            KafkaMessage message = new KafkaMessage(record.value(), record.offset());
            messages.add(message);
        }
        return messages;
    }
    return new ArrayList<>();
}