Examples with RowChange com.alibaba.otter.canal.protocol.CanalEntry.RowChange used on opensource projects

Example 11 with RowChange

use of com.alibaba.otter.canal.protocol.CanalEntry.RowChange in project canal by alibaba.

the class MQMessageUtils method buildMessageData.

/**
 * 多线程构造message的rowChanged对象，比如为partition/flastMessage转化等处理 </br>
 * 因为protobuf对象的序列化和反序列化是cpu密集型，串行执行会有代价
 */
public static EntryRowData[] buildMessageData(Message message, ThreadPoolExecutor executor) {
    ExecutorTemplate template = new ExecutorTemplate(executor);
    if (message.isRaw()) {
        List<ByteString> rawEntries = message.getRawEntries();
        final EntryRowData[] datas = new EntryRowData[rawEntries.size()];
        int i = 0;
        for (ByteString byteString : rawEntries) {
            final int index = i;
            template.submit(() -> {
                try {
                    Entry entry = Entry.parseFrom(byteString);
                    RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
                    datas[index] = new EntryRowData();
                    datas[index].entry = entry;
                    datas[index].rowChange = rowChange;
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException(e);
                }
            });
            i++;
        }
        template.waitForResult();
        return datas;
    } else {
        final EntryRowData[] datas = new EntryRowData[message.getEntries().size()];
        int i = 0;
        for (Entry entry : message.getEntries()) {
            final int index = i;
            template.submit(() -> {
                try {
                    RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
                    datas[index] = new EntryRowData();
                    datas[index].entry = entry;
                    datas[index].rowChange = rowChange;
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException(e);
                }
            });
            i++;
        }
        template.waitForResult();
        return datas;
    }
}

Example 12 with RowChange

use of com.alibaba.otter.canal.protocol.CanalEntry.RowChange in project canal by alibaba.

the class MQMessageUtils method messagePartition.

/**
 * 将 message 分区
 *
 * @param partitionsNum 分区数
 * @param pkHashConfigs 分区库表主键正则表达式
 * @param databaseHash 是否取消根据database进行hash
 * @return 分区message数组
 */
@SuppressWarnings("unchecked")
public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs, boolean databaseHash) {
    if (partitionsNum == null) {
        partitionsNum = 1;
    }
    Message[] partitionMessages = new Message[partitionsNum];
    List<Entry>[] partitionEntries = new List[partitionsNum];
    for (int i = 0; i < partitionsNum; i++) {
        // 注意一下并发
        partitionEntries[i] = Collections.synchronizedList(new ArrayList<>());
    }
    for (EntryRowData data : datas) {
        CanalEntry.Entry entry = data.entry;
        CanalEntry.RowChange rowChange = data.rowChange;
        // 如果有分区路由,则忽略begin/end事件
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
            continue;
        }
        if (rowChange.getIsDdl()) {
            partitionEntries[0].add(entry);
        } else {
            if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
                String database = entry.getHeader().getSchemaName();
                String table = entry.getHeader().getTableName();
                HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
                if (hashMode == null) {
                    // 如果都没有匹配，发送到第一个分区
                    partitionEntries[0].add(entry);
                } else if (hashMode.tableHash) {
                    int hashCode = table.hashCode();
                    int pkHash = Math.abs(hashCode) % partitionsNum;
                    pkHash = Math.abs(pkHash);
                    // tableHash not need split entry message
                    partitionEntries[pkHash].add(entry);
                } else {
                    // build new entry
                    Entry.Builder builder = Entry.newBuilder(entry);
                    RowChange.Builder rowChangeBuilder = RowChange.newBuilder(rowChange);
                    for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
                        int hashCode = 0;
                        if (databaseHash) {
                            hashCode = database.hashCode();
                        }
                        CanalEntry.EventType eventType = rowChange.getEventType();
                        List<CanalEntry.Column> columns = null;
                        if (eventType == CanalEntry.EventType.DELETE) {
                            columns = rowData.getBeforeColumnsList();
                        } else {
                            columns = rowData.getAfterColumnsList();
                        }
                        if (hashMode.autoPkHash) {
                            // isEmpty use default pkNames
                            for (CanalEntry.Column column : columns) {
                                if (column.getIsKey()) {
                                    hashCode = hashCode ^ column.getValue().hashCode();
                                }
                            }
                        } else {
                            for (CanalEntry.Column column : columns) {
                                if (checkPkNamesHasContain(hashMode.pkNames, column.getName())) {
                                    hashCode = hashCode ^ column.getValue().hashCode();
                                }
                            }
                        }
                        int pkHash = Math.abs(hashCode) % partitionsNum;
                        pkHash = Math.abs(pkHash);
                        // clear rowDatas
                        rowChangeBuilder.clearRowDatas();
                        rowChangeBuilder.addRowDatas(rowData);
                        builder.clearStoreValue();
                        builder.setStoreValue(rowChangeBuilder.build().toByteString());
                        partitionEntries[pkHash].add(builder.build());
                    }
                }
            } else {
                // 针对stmt/mixed binlog格式的query事件
                partitionEntries[0].add(entry);
            }
        }
    }
    for (int i = 0; i < partitionsNum; i++) {
        List<Entry> entriesTmp = partitionEntries[i];
        if (!entriesTmp.isEmpty()) {
            partitionMessages[i] = new Message(id, entriesTmp);
        }
    }
    return partitionMessages;
}