Examples with FlatMessage com.alibaba.otter.canal.protocol.FlatMessage used on opensource projects

Example 11 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class KafkaCanalConnector method getFlatListWithoutAck.

@Override
public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return Lists.newArrayList();
    }
    ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
    currentOffsets.clear();
    for (TopicPartition topicPartition : records.partitions()) {
        currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition));
    }
    if (!records.isEmpty()) {
        List<FlatMessage> flatMessages = new ArrayList<>();
        for (ConsumerRecord<String, String> record : records) {
            String flatMessageJson = record.value();
            FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
            flatMessages.add(flatMessage);
        }
        return flatMessages;
    }
    return Lists.newArrayList();
}

Example 12 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class RabbitMQCanalConnector method process.

private boolean process(byte[] messageData) {
    if (logger.isDebugEnabled()) {
        logger.debug("Get Message: {}", new String(messageData));
    }
    List messageList = new ArrayList<>();
    if (!flatMessage) {
        Message message = CanalMessageDeserializer.deserializer(messageData);
        messageList.add(message);
    } else {
        FlatMessage flatMessage = JSON.parseObject(messageData, FlatMessage.class);
        messageList.add(flatMessage);
    }
    ConsumerBatchMessage batchMessage;
    if (!flatMessage) {
        batchMessage = new ConsumerBatchMessage<Message>(messageList);
    } else {
        batchMessage = new ConsumerBatchMessage<FlatMessage>(messageList);
    }
    try {
        messageBlockingQueue.put(batchMessage);
    } catch (InterruptedException e) {
        logger.error("Put message to queue error", e);
        throw new RuntimeException(e);
    }
    boolean isCompleted;
    try {
        isCompleted = batchMessage.waitFinish(batchProcessTimeout);
    } catch (InterruptedException e) {
        logger.error("Interrupted when waiting messages to be finished.", e);
        throw new RuntimeException(e);
    }
    boolean isSuccess = batchMessage.isSuccess();
    return isCompleted && isSuccess;
}

Example 13 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class CanalKafkaClientFlatMessageExample method process.

private void process() {
    while (!running) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        }
    }
    while (running) {
        try {
            connector.connect();
            connector.subscribe();
            while (running) {
                try {
                    // 获取message
                    List<FlatMessage> messages = connector.getFlatList(100L, TimeUnit.MILLISECONDS);
                    if (messages == null) {
                        continue;
                    }
                    for (FlatMessage message : messages) {
                        long batchId = message.getId();
                        if (batchId == -1 || message.getData() == null) {
                        // try {
                        // Thread.sleep(1000);
                        // } catch (InterruptedException e) {
                        // }
                        } else {
                            // printSummary(message, batchId, size);
                            // printEntry(message.getEntries());
                            logger.info(message.toString());
                        }
                    }
                    // 提交确认
                    connector.ack();
                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                }
            }
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
        }
    }
    connector.unsubscribe();
    connector.disconnect();
}