Examples with CommonMessage com.alibaba.otter.canal.connector.core.consumer.CommonMessage used on opensource projects

Example 1 with CommonMessage

use of com.alibaba.otter.canal.connector.core.consumer.CommonMessage in project canal by alibaba.

the class CanalKafkaConsumer method getMessage.

@SuppressWarnings("unchecked")
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
    if (!flatMessage) {
        ConsumerRecords<String, Message> records = (ConsumerRecords<String, Message>) kafkaConsumer.poll(unit.toMillis(timeout));
        if (!records.isEmpty()) {
            currentOffsets.clear();
            List<CommonMessage> messages = new ArrayList<>();
            for (ConsumerRecord<String, Message> record : records) {
                if (currentOffsets.get(record.partition()) == null) {
                    currentOffsets.put(record.partition(), record.offset());
                }
                messages.addAll(MessageUtil.convert(record.value()));
            }
            return messages;
        }
    } else {
        ConsumerRecords<String, String> records = (ConsumerRecords<String, String>) kafkaConsumer.poll(unit.toMillis(timeout));
        if (!records.isEmpty()) {
            List<CommonMessage> messages = new ArrayList<>();
            currentOffsets.clear();
            for (ConsumerRecord<String, String> record : records) {
                if (currentOffsets.get(record.partition()) == null) {
                    currentOffsets.put(record.partition(), record.offset());
                }
                String flatMessageJson = record.value();
                CommonMessage flatMessages = JSON.parseObject(flatMessageJson, CommonMessage.class);
                messages.add(flatMessages);
            }
            return messages;
        }
    }
    return null;
}

Example 2 with CommonMessage

use of com.alibaba.otter.canal.connector.core.consumer.CommonMessage in project canal by alibaba.

the class CanalPulsarMQConsumerTest method getMessage.

@Test
public void getMessage() {
    consumer.connect();
    while (true) {
        List<CommonMessage> list = consumer.getMessage(-1L, TimeUnit.SECONDS);
        String time = LocalTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss"));
        if (null == list || list.isEmpty()) {
            System.out.println(time + " Receive empty");
            continue;
        }
        for (CommonMessage m : list) {
            System.out.println(time + " Receive ==> " + JSON.toJSONString(m));
        }
        consumer.ack();
    }
}

Example 3 with CommonMessage

use of com.alibaba.otter.canal.connector.core.consumer.CommonMessage in project canal by alibaba.

the class CanalRocketMQConsumer method process.

private boolean process(List<MessageExt> messageExts) {
    if (logger.isDebugEnabled()) {
        logger.debug("Get Message: {}", messageExts);
    }
    List<CommonMessage> messageList = new ArrayList<>();
    for (MessageExt messageExt : messageExts) {
        byte[] data = messageExt.getBody();
        if (data != null) {
            try {
                if (!flatMessage) {
                    Message message = CanalMessageSerializerUtil.deserializer(data);
                    messageList.addAll(MessageUtil.convert(message));
                } else {
                    CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
                    messageList.add(commonMessage);
                }
            } catch (Exception ex) {
                logger.error("Add message error", ex);
                throw new CanalClientException(ex);
            }
        } else {
            logger.warn("Received message data is null");
        }
    }
    ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
    try {
        messageBlockingQueue.put(batchMessage);
    } catch (InterruptedException e) {
        logger.error("Put message to queue error", e);
        throw new RuntimeException(e);
    }
    boolean isCompleted;
    try {
        isCompleted = batchMessage.waitFinish(batchProcessTimeout);
    } catch (InterruptedException e) {
        logger.error("Interrupted when waiting messages to be finished.", e);
        throw new RuntimeException(e);
    }
    boolean isSuccess = batchMessage.isSuccess();
    return isCompleted && isSuccess;
}

Example 4 with CommonMessage

use of com.alibaba.otter.canal.connector.core.consumer.CommonMessage in project canal by alibaba.

the class CanalTCPConsumer method getMessage.

@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
    try {
        Message message = canalConnector.getWithoutAck(batchSize, timeout, unit);
        long batchId = message.getId();
        currentBatchId = batchId;
        int size = message.getEntries().size();
        if (batchId == -1 || size == 0) {
            return null;
        } else {
            return MessageUtil.convert(message);
        }
    } catch (Throwable e) {
        throw new RuntimeException(e);
    }
}

Example 5 with CommonMessage

use of com.alibaba.otter.canal.connector.core.consumer.CommonMessage in project canal by alibaba.

the class CanalPulsarMQConsumer method getMessage.

@SuppressWarnings("unchecked")
@Override
public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
    List<CommonMessage> messageList = Lists.newArrayList();
    try {
        Messages<byte[]> messages = pulsarMQConsumer.batchReceive();
        if (null == messages || messages.size() == 0) {
            return messageList;
        }
        // 保存当前消费记录，用于ack和rollback
        this.lastGetBatchMessage = messages;
        for (org.apache.pulsar.client.api.Message<byte[]> msg : messages) {
            byte[] data = msg.getData();
            if (!this.flatMessage) {
                Message message = CanalMessageSerializerUtil.deserializer(data);
                List<CommonMessage> list = MessageUtil.convert(message);
                messageList.addAll(list);
            } else {
                CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
                messageList.add(commonMessage);
            }
        }
    } catch (PulsarClientException e) {
        throw new CanalClientException("Receive pulsar batch message error", e);
    }
    return messageList;
}