Examples with FlatMessage com.alibaba.otter.canal.protocol.FlatMessage used on opensource projects

Example 6 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class RocketMQCanalConnector method process.

private boolean process(List<MessageExt> messageExts) {
    if (logger.isDebugEnabled()) {
        logger.debug("Get Message: {}", messageExts);
    }
    List messageList = new ArrayList<>();
    for (MessageExt messageExt : messageExts) {
        byte[] data = messageExt.getBody();
        if (data != null) {
            try {
                if (!flatMessage) {
                    Message message = CanalMessageDeserializer.deserializer(data);
                    messageList.add(message);
                } else {
                    FlatMessage flatMessage = JSON.parseObject(data, FlatMessage.class);
                    messageList.add(flatMessage);
                }
            } catch (Exception ex) {
                logger.error("Add message error", ex);
                throw new CanalClientException(ex);
            }
        } else {
            logger.warn("Received message data is null");
        }
    }
    ConsumerBatchMessage batchMessage;
    if (!flatMessage) {
        batchMessage = new ConsumerBatchMessage<Message>(messageList);
    } else {
        batchMessage = new ConsumerBatchMessage<FlatMessage>(messageList);
    }
    try {
        messageBlockingQueue.put(batchMessage);
    } catch (InterruptedException e) {
        logger.error("Put message to queue error", e);
        throw new RuntimeException(e);
    }
    boolean isCompleted;
    try {
        isCompleted = batchMessage.waitFinish(batchProcessTimeout);
    } catch (InterruptedException e) {
        logger.error("Interrupted when waiting messages to be finished.", e);
        throw new RuntimeException(e);
    }
    boolean isSuccess = batchMessage.isSuccess();
    return isCompleted && isSuccess;
}

Example 7 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class CanalRocketMQProducer method send.

public void send(final MQDestination destination, String topicName, com.alibaba.otter.canal.protocol.Message message) {
    // 获取当前topic的分区数
    Integer partitionNum = MQMessageUtils.parseDynamicTopicPartition(topicName, destination.getDynamicTopicPartitionNum());
    // 获取topic的队列数为分区数
    if (partitionNum == null) {
        partitionNum = getTopicDynamicQueuesSize(destination.getEnableDynamicQueuePartition(), topicName);
    }
    if (partitionNum == null) {
        partitionNum = destination.getPartitionsNum();
    }
    if (!mqProperties.isFlatMessage()) {
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            // 并发构造
            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
            // 串行分区
            com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas, message.getId(), partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
            int length = messages.length;
            ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
            for (int i = 0; i < length; i++) {
                com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
                if (dataPartition != null) {
                    final int index = i;
                    template.submit(() -> {
                        Message data = new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), CanalMessageSerializerUtil.serializer(dataPartition, mqProperties.isFilterTransactionEntry()));
                        sendMessage(data, index);
                    });
                }
            }
            // 等所有分片发送完毕
            template.waitForResult();
        } else {
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            Message data = new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
            sendMessage(data, partition);
        }
    } else {
        // 并发构造
        MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, buildExecutor);
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
        // 初始化分区合并队列
        if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
            List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
            for (int i = 0; i < partitionNum; i++) {
                partitionFlatMessages.add(new ArrayList<>());
            }
            for (FlatMessage flatMessage : flatMessages) {
                FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage, partitionNum, destination.getPartitionHash(), mqProperties.isDatabaseHash());
                int length = partitionFlatMessage.length;
                for (int i = 0; i < length; i++) {
                    // 增加null判断,issue #3267
                    if (partitionFlatMessage[i] != null) {
                        partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
                    }
                }
            }
            ExecutorTemplate template = new ExecutorTemplate(sendPartitionExecutor);
            for (int i = 0; i < partitionFlatMessages.size(); i++) {
                final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
                if (flatMessagePart != null && flatMessagePart.size() > 0) {
                    final int index = i;
                    template.submit(() -> {
                        List<Message> messages = flatMessagePart.stream().map(flatMessage -> new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))).collect(Collectors.toList());
                        // 批量发送
                        sendMessage(messages, index);
                    });
                }
            }
            // 批量等所有分区的结果
            template.waitForResult();
        } else {
            final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
            List<Message> messages = flatMessages.stream().map(flatMessage -> new Message(topicName, ((RocketMQProducerConfig) this.mqProperties).getTag(), JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue))).collect(Collectors.toList());
            // 批量发送
            sendMessage(messages, partition);
        }
    }
}

Example 8 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class MQMessageUtils method messageConverter.

/**
 * 将Message转换为FlatMessage
 *
 * @return FlatMessage列表
 * @author agapple 2018年12月11日 下午1:28:32
 */
public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) {
    List<FlatMessage> flatMessages = new ArrayList<>();
    for (EntryRowData entryRowData : datas) {
        CanalEntry.Entry entry = entryRowData.entry;
        CanalEntry.RowChange rowChange = entryRowData.rowChange;
        // 如果有分区路由,则忽略begin/end事件
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
            continue;
        }
        // build flatMessage
        CanalEntry.EventType eventType = rowChange.getEventType();
        FlatMessage flatMessage = new FlatMessage(id);
        flatMessages.add(flatMessage);
        flatMessage.setDatabase(entry.getHeader().getSchemaName());
        flatMessage.setTable(entry.getHeader().getTableName());
        flatMessage.setIsDdl(rowChange.getIsDdl());
        flatMessage.setType(eventType.toString());
        flatMessage.setEs(entry.getHeader().getExecuteTime());
        flatMessage.setTs(System.currentTimeMillis());
        flatMessage.setSql(rowChange.getSql());
        if (!rowChange.getIsDdl()) {
            Map<String, Integer> sqlType = new LinkedHashMap<>();
            Map<String, String> mysqlType = new LinkedHashMap<>();
            List<Map<String, String>> data = new ArrayList<>();
            List<Map<String, String>> old = new ArrayList<>();
            Set<String> updateSet = new HashSet<>();
            boolean hasInitPkNames = false;
            for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
                if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE && eventType != CanalEntry.EventType.DELETE) {
                    continue;
                }
                Map<String, String> row = new LinkedHashMap<>();
                List<CanalEntry.Column> columns;
                if (eventType == CanalEntry.EventType.DELETE) {
                    columns = rowData.getBeforeColumnsList();
                } else {
                    columns = rowData.getAfterColumnsList();
                }
                for (CanalEntry.Column column : columns) {
                    if (!hasInitPkNames && column.getIsKey()) {
                        flatMessage.addPkName(column.getName());
                    }
                    sqlType.put(column.getName(), column.getSqlType());
                    mysqlType.put(column.getName(), column.getMysqlType());
                    if (column.getIsNull()) {
                        row.put(column.getName(), null);
                    } else {
                        row.put(column.getName(), column.getValue());
                    }
                    // 获取update为true的字段
                    if (column.getUpdated()) {
                        updateSet.add(column.getName());
                    }
                }
                hasInitPkNames = true;
                if (!row.isEmpty()) {
                    data.add(row);
                }
                if (eventType == CanalEntry.EventType.UPDATE) {
                    Map<String, String> rowOld = new LinkedHashMap<>();
                    for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
                        if (updateSet.contains(column.getName())) {
                            if (column.getIsNull()) {
                                rowOld.put(column.getName(), null);
                            } else {
                                rowOld.put(column.getName(), column.getValue());
                            }
                        }
                    }
                    // update操作将记录修改前的值
                    if (!rowOld.isEmpty()) {
                        old.add(rowOld);
                    }
                }
            }
            if (!sqlType.isEmpty()) {
                flatMessage.setSqlType(sqlType);
            }
            if (!mysqlType.isEmpty()) {
                flatMessage.setMysqlType(mysqlType);
            }
            if (!data.isEmpty()) {
                flatMessage.setData(data);
            }
            if (!old.isEmpty()) {
                flatMessage.setOld(old);
            }
        }
    }
    return flatMessages;
}

Example 9 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class MQMessageUtils method messagePartition.

/**
 * 将FlatMessage按指定的字段值hash拆分
 *
 * @param flatMessage flatMessage
 * @param partitionsNum 分区数量
 * @param pkHashConfigs hash映射
 * @param databaseHash 是否取消根据database进行hash
 * @return 拆分后的flatMessage数组
 */
public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum, String pkHashConfigs, boolean databaseHash) {
    if (partitionsNum == null) {
        partitionsNum = 1;
    }
    FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
    if (flatMessage.getIsDdl()) {
        partitionMessages[0] = flatMessage;
    } else {
        if (flatMessage.getData() != null && !flatMessage.getData().isEmpty()) {
            String database = flatMessage.getDatabase();
            String table = flatMessage.getTable();
            HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
            if (hashMode == null) {
                // 如果都没有匹配，发送到第一个分区
                partitionMessages[0] = flatMessage;
            } else if (hashMode.tableHash) {
                int hashCode = table.hashCode();
                int pkHash = Math.abs(hashCode) % partitionsNum;
                // math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
                pkHash = Math.abs(pkHash);
                partitionMessages[pkHash] = flatMessage;
            } else {
                List<String> pkNames = hashMode.pkNames;
                if (hashMode.autoPkHash) {
                    pkNames = flatMessage.getPkNames();
                }
                int idx = 0;
                for (Map<String, String> row : flatMessage.getData()) {
                    int hashCode = 0;
                    if (databaseHash) {
                        hashCode = database.hashCode();
                    }
                    if (pkNames != null) {
                        for (String pkName : pkNames) {
                            String value = row.get(pkName);
                            if (value == null) {
                                value = "";
                            }
                            hashCode = hashCode ^ value.hashCode();
                        }
                    }
                    int pkHash = Math.abs(hashCode) % partitionsNum;
                    // math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
                    pkHash = Math.abs(pkHash);
                    FlatMessage flatMessageTmp = partitionMessages[pkHash];
                    if (flatMessageTmp == null) {
                        flatMessageTmp = new FlatMessage(flatMessage.getId());
                        partitionMessages[pkHash] = flatMessageTmp;
                        flatMessageTmp.setDatabase(flatMessage.getDatabase());
                        flatMessageTmp.setTable(flatMessage.getTable());
                        flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
                        flatMessageTmp.setType(flatMessage.getType());
                        flatMessageTmp.setSql(flatMessage.getSql());
                        flatMessageTmp.setSqlType(flatMessage.getSqlType());
                        flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
                        flatMessageTmp.setEs(flatMessage.getEs());
                        flatMessageTmp.setTs(flatMessage.getTs());
                        flatMessageTmp.setPkNames(flatMessage.getPkNames());
                    }
                    List<Map<String, String>> data = flatMessageTmp.getData();
                    if (data == null) {
                        data = new ArrayList<>();
                        flatMessageTmp.setData(data);
                    }
                    data.add(row);
                    if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
                        List<Map<String, String>> old = flatMessageTmp.getOld();
                        if (old == null) {
                            old = new ArrayList<>();
                            flatMessageTmp.setOld(old);
                        }
                        old.add(flatMessage.getOld().get(idx));
                    }
                    idx++;
                }
            }
        } else {
            // 针对stmt/mixed binlog格式的query事件
            partitionMessages[0] = flatMessage;
        }
    }
    return partitionMessages;
}

Example 10 with FlatMessage

use of com.alibaba.otter.canal.protocol.FlatMessage in project canal by alibaba.

the class CanalRabbitMQProducer method send.

private void send(MQDestination canalDestination, String topicName, Message messageSub) {
    if (!mqProperties.isFlatMessage()) {
        byte[] message = CanalMessageSerializerUtil.serializer(messageSub, mqProperties.isFilterTransactionEntry());
        if (logger.isDebugEnabled()) {
            logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
        }
        sendMessage(topicName, message);
    } else {
        // 并发构造
        MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, buildExecutor);
        // 串行分区
        List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
        for (FlatMessage flatMessage : flatMessages) {
            byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
            if (logger.isDebugEnabled()) {
                logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
            }
            sendMessage(topicName, message);
        }
    }
}