Examples with KafkaMessage com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage used on opensource projects

Example 1 with KafkaMessage

use of com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage in project canal by alibaba.

the class CanalKafkaOffsetClientExample method process.

private void process() {
    while (!running) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        }
    }
    while (running) {
        try {
            // 修改 AutoOffsetReset 的值，默认（earliest）
            // connector.setAutoOffsetReset(null);
            connector.connect();
            connector.subscribe();
            // 消息起始偏移地址
            long offset = -1;
            // 错误次数
            int errorCount = 0;
            while (running) {
                try {
                    // 错误重试次数超过3次后，每30秒递增重试
                    if (errorCount > 2) {
                        Thread.sleep((errorCount - 2) * 1000 * 30);
                    }
                    // 获取message
                    List<KafkaMessage> messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS, offset);
                    if (messages == null) {
                        continue;
                    }
                    for (KafkaMessage message : messages) {
                        long batchId = message.getId();
                        int size = message.getEntries().size();
                        if (batchId == -1 || size == 0) {
                            continue;
                        }
                        // 记录第一条消息的offset，用于处理数据异常时重新从此位置获取消息
                        if (offset < 0) {
                            offset = message.getOffset();
                        }
                        // printSummary(message, batchId, size);
                        // printEntry(message.getEntries());
                        logger.info(message.toString());
                    }
                    // 提交确认
                    connector.ack();
                    // 还原offset
                    offset = -1;
                    errorCount = 0;
                } catch (Exception e) {
                    errorCount++;
                    logger.error(e.getMessage(), e);
                    if (errorCount == 3) {
                    // 重试3次后发送邮件提醒异常
                    // mailService.sendMail("同步数据异常，请及时处理", "错误消息");
                    }
                }
            }
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
        }
    }
    connector.unsubscribe();
    connector.disconnect();
}

Example 2 with KafkaMessage

use of com.alibaba.otter.canal.client.kafka.protocol.KafkaMessage in project canal by alibaba.

the class KafkaOffsetCanalConnector method getListWithoutAck.

/**
 * 获取Kafka消息，不确认
 *
 * @param timeout
 * @param unit
 * @param offset  消息偏移地址（-1为不偏移）
 * @return
 * @throws CanalClientException
 */
public List<KafkaMessage> getListWithoutAck(Long timeout, TimeUnit unit, long offset) throws CanalClientException {
    waitClientRunning();
    if (!running) {
        return new ArrayList<>();
    }
    if (offset > -1) {
        TopicPartition tp = new TopicPartition(topic, partition == null ? 0 : partition);
        kafkaConsumer.seek(tp, offset);
    }
    ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
    if (!records.isEmpty()) {
        List<KafkaMessage> messages = new ArrayList<>();
        for (ConsumerRecord<String, Message> record : records) {
            KafkaMessage message = new KafkaMessage(record.value(), record.offset());
            messages.add(message);
        }
        return messages;
    }
    return new ArrayList<>();
}