Examples with Message com.alibaba.otter.canal.protocol.Message used on opensource projects

Example 21 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class BaseCanalServerWithEmbededTest method testRollback.

// @Test
public void testRollback() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.getWithoutAck(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
        }
    }
    System.out.println("!!!!!! testRollback totalCount : " + totalCount);
    // 直接rollback掉，再取一次
    server.rollback(clientIdentity);
    emptyCount = 0;
    totalCount = 0;
    while (emptyCount < maxEmptyCount) {
        Message message = server.getWithoutAck(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
        }
    }
    System.out.println("!!!!!! testRollback after rollback ,  totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}

Example 22 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class BaseCanalServerWithEmbededTest method testGetWithoutAck.

@Test
public void testGetWithoutAck() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.getWithoutAck(clientIdentity, 11);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
            server.ack(clientIdentity, message.getId());
        }
    }
    System.out.println("!!!!!! testGetWithoutAck totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}

Example 23 with Message

use of com.alibaba.otter.canal.protocol.Message in project java-example by 1479005017.

the class CanalTest method test.

@Test
public void test() {
    // 创建链接
    CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(AddressUtils.getHostIp(), 11111), "example", "", "");
    try {
        connector.connect();
        connector.subscribe(".*\\.user");
        while (true) {
            // 获取指定数量的数据
            Message message = connector.getWithoutAck(1000);
            try {
                if (message.getId() == -1 || message.getEntries().size() == 0) {
                    Thread.sleep(3000);
                    continue;
                }
                CanalPrinter.printSummary(message);
                CanalPrinter.printEntry(message);
                // 提交确认
                connector.ack(message.getId());
            } catch (Exception e) {
                e.printStackTrace();
                // 处理失败，回滚数据
                connector.rollback(message.getId());
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        connector.disconnect();
    }
}

Example 24 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class MQMessageUtils method messagePartition.

/**
 * 将 message 分区
 *
 * @param partitionsNum 分区数
 * @param pkHashConfigs 分区库表主键正则表达式
 * @param databaseHash 是否取消根据database进行hash
 * @return 分区message数组
 */
@SuppressWarnings("unchecked")
public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs, boolean databaseHash) {
    if (partitionsNum == null) {
        partitionsNum = 1;
    }
    Message[] partitionMessages = new Message[partitionsNum];
    List<Entry>[] partitionEntries = new List[partitionsNum];
    for (int i = 0; i < partitionsNum; i++) {
        // 注意一下并发
        partitionEntries[i] = Collections.synchronizedList(new ArrayList<>());
    }
    for (EntryRowData data : datas) {
        CanalEntry.Entry entry = data.entry;
        CanalEntry.RowChange rowChange = data.rowChange;
        // 如果有分区路由,则忽略begin/end事件
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
            continue;
        }
        if (rowChange.getIsDdl()) {
            partitionEntries[0].add(entry);
        } else {
            if (rowChange.getRowDatasList() != null && !rowChange.getRowDatasList().isEmpty()) {
                String database = entry.getHeader().getSchemaName();
                String table = entry.getHeader().getTableName();
                HashMode hashMode = getPartitionHashColumns(database + "." + table, pkHashConfigs);
                if (hashMode == null) {
                    // 如果都没有匹配，发送到第一个分区
                    partitionEntries[0].add(entry);
                } else if (hashMode.tableHash) {
                    int hashCode = table.hashCode();
                    int pkHash = Math.abs(hashCode) % partitionsNum;
                    pkHash = Math.abs(pkHash);
                    // tableHash not need split entry message
                    partitionEntries[pkHash].add(entry);
                } else {
                    // build new entry
                    Entry.Builder builder = Entry.newBuilder(entry);
                    RowChange.Builder rowChangeBuilder = RowChange.newBuilder(rowChange);
                    for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
                        int hashCode = 0;
                        if (databaseHash) {
                            hashCode = database.hashCode();
                        }
                        CanalEntry.EventType eventType = rowChange.getEventType();
                        List<CanalEntry.Column> columns = null;
                        if (eventType == CanalEntry.EventType.DELETE) {
                            columns = rowData.getBeforeColumnsList();
                        } else {
                            columns = rowData.getAfterColumnsList();
                        }
                        if (hashMode.autoPkHash) {
                            // isEmpty use default pkNames
                            for (CanalEntry.Column column : columns) {
                                if (column.getIsKey()) {
                                    hashCode = hashCode ^ column.getValue().hashCode();
                                }
                            }
                        } else {
                            for (CanalEntry.Column column : columns) {
                                if (checkPkNamesHasContain(hashMode.pkNames, column.getName())) {
                                    hashCode = hashCode ^ column.getValue().hashCode();
                                }
                            }
                        }
                        int pkHash = Math.abs(hashCode) % partitionsNum;
                        pkHash = Math.abs(pkHash);
                        // clear rowDatas
                        rowChangeBuilder.clearRowDatas();
                        rowChangeBuilder.addRowDatas(rowData);
                        builder.clearStoreValue();
                        builder.setStoreValue(rowChangeBuilder.build().toByteString());
                        partitionEntries[pkHash].add(builder.build());
                    }
                }
            } else {
                // 针对stmt/mixed binlog格式的query事件
                partitionEntries[0].add(entry);
            }
        }
    }
    for (int i = 0; i < partitionsNum; i++) {
        List<Entry> entriesTmp = partitionEntries[i];
        if (!entriesTmp.isEmpty()) {
            partitionMessages[i] = new Message(id, entriesTmp);
        }
    }
    return partitionMessages;
}

Example 25 with Message

use of com.alibaba.otter.canal.protocol.Message in project canal by alibaba.

the class MQMessageUtils method put2MapMessage.

private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName, CanalEntry.Entry entry) {
    Message message = messageMap.get(topicName);
    if (message == null) {
        message = new Message(messageId, new ArrayList<>());
        messageMap.put(topicName, message);
    }
    message.getEntries().add(entry);
}