Examples with RowChange com.alibaba.otter.canal.protocol.CanalEntry.RowChange used on opensource projects

Example 1 with RowChange

use of com.alibaba.otter.canal.protocol.CanalEntry.RowChange in project canal by alibaba.

the class AbstractCanalClientTest method printEntry.

protected void printEntry(List<Entry> entrys) {
    for (Entry entry : entrys) {
        long executeTime = entry.getHeader().getExecuteTime();
        long delayTime = new Date().getTime() - executeTime;
        if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
                TransactionBegin begin = null;
                try {
                    begin = TransactionBegin.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务头信息，执行的线程id，事务耗时
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
                logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
            } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
                TransactionEnd end = null;
                try {
                    end = TransactionEnd.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务提交信息，事务id
                logger.info("----------------\n");
                logger.info(" END ----> transaction id: {}", end.getTransactionId());
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
            }
            continue;
        }
        if (entry.getEntryType() == EntryType.ROWDATA) {
            RowChange rowChage = null;
            try {
                rowChage = RowChange.parseFrom(entry.getStoreValue());
            } catch (Exception e) {
                throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
            }
            EventType eventType = rowChage.getEventType();
            logger.info(row_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType, String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
            if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
                logger.info(" sql ----> " + rowChage.getSql() + SEP);
                continue;
            }
            for (RowData rowData : rowChage.getRowDatasList()) {
                if (eventType == EventType.DELETE) {
                    printColumn(rowData.getBeforeColumnsList());
                } else if (eventType == EventType.INSERT) {
                    printColumn(rowData.getAfterColumnsList());
                } else {
                    printColumn(rowData.getAfterColumnsList());
                }
            }
        }
    }
}

Example 2 with RowChange

use of com.alibaba.otter.canal.protocol.CanalEntry.RowChange in project otter by alibaba.

the class MessageParser method parse.

/**
 * 将对应canal送出来的Entry对象解析为otter使用的内部对象
 *
 * <pre>
 * 需要处理数据过滤：
 * 1. Transaction Begin/End过滤
 * 2. retl.retl_client/retl.retl_mark 回环标记处理以及后续的回环数据过滤
 * 3. retl.xdual canal心跳表数据过滤
 * </pre>
 */
public List<EventData> parse(Long pipelineId, List<Entry> datas) throws SelectException {
    List<EventData> eventDatas = new ArrayList<EventData>();
    Pipeline pipeline = configClientService.findPipeline(pipelineId);
    List<Entry> transactionDataBuffer = new ArrayList<Entry>();
    // hz为主站点，us->hz的数据，需要回环同步会us。并且需要开启回环补救算法
    PipelineParameter pipelineParameter = pipeline.getParameters();
    boolean enableLoopbackRemedy = pipelineParameter.isEnableRemedy() && pipelineParameter.isHome() && pipelineParameter.getRemedyAlgorithm().isLoopback();
    boolean isLoopback = false;
    // 判断是否属于需要loopback处理的类型，只处理正常otter同步产生的回环数据，因为会有业务方手工屏蔽同步的接口，避免回环
    boolean needLoopback = false;
    long now = new Date().getTime();
    try {
        for (Entry entry : datas) {
            switch(entry.getEntryType()) {
                case TRANSACTIONBEGIN:
                    isLoopback = false;
                    break;
                case ROWDATA:
                    String tableName = entry.getHeader().getTableName();
                    // 判断是否是回环表retl_mark
                    boolean isMarkTable = tableName.equalsIgnoreCase(pipeline.getParameters().getSystemMarkTable());
                    if (isMarkTable) {
                        RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
                        if (!rowChange.getIsDdl()) {
                            int loopback = 0;
                            if (rowChange.getRowDatasCount() > 0) {
                                loopback = checkLoopback(pipeline, rowChange.getRowDatas(0));
                            }
                            if (loopback == 2) {
                                // 只处理正常同步产生的回环数据
                                needLoopback |= true;
                            }
                            isLoopback |= loopback > 0;
                        }
                    }
                    // 检查下otter3.0的回环表，对应的schmea会比较随意，所以不做比较
                    boolean isCompatibleLoopback = tableName.equalsIgnoreCase(compatibleMarkTable);
                    if (isCompatibleLoopback) {
                        RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
                        if (!rowChange.getIsDdl()) {
                            int loopback = 0;
                            if (rowChange.getRowDatasCount() > 0) {
                                loopback = checkCompatibleLoopback(pipeline, rowChange.getRowDatas(0));
                            }
                            if (loopback == 2) {
                                // 只处理正常同步产生的回环数据
                                needLoopback |= true;
                            }
                            isLoopback |= loopback > 0;
                        }
                    }
                    if ((!isLoopback || (enableLoopbackRemedy && needLoopback)) && !isMarkTable && !isCompatibleLoopback) {
                        transactionDataBuffer.add(entry);
                    }
                    break;
                case TRANSACTIONEND:
                    if (!isLoopback || (enableLoopbackRemedy && needLoopback)) {
                        // 添加数据解析
                        for (Entry bufferEntry : transactionDataBuffer) {
                            List<EventData> parseDatas = internParse(pipeline, bufferEntry);
                            if (CollectionUtils.isEmpty(parseDatas)) {
                                // 可能为空，针对ddl返回时就为null
                                continue;
                            }
                            // 初步计算一下事件大小
                            long totalSize = bufferEntry.getHeader().getEventLength();
                            long eachSize = totalSize / parseDatas.size();
                            for (EventData eventData : parseDatas) {
                                if (eventData == null) {
                                    continue;
                                }
                                // 记录一下大小
                                eventData.setSize(eachSize);
                                if (needLoopback) {
                                    // 如果延迟超过指定的阀值，则设置为需要反查db
                                    if (now - eventData.getExecuteTime() > 1000 * pipeline.getParameters().getRemedyDelayThresoldForMedia()) {
                                        eventData.setSyncConsistency(SyncConsistency.MEDIA);
                                    } else {
                                        eventData.setSyncConsistency(SyncConsistency.BASE);
                                    }
                                    eventData.setRemedy(true);
                                }
                                eventDatas.add(eventData);
                            }
                        }
                    }
                    isLoopback = false;
                    needLoopback = false;
                    transactionDataBuffer.clear();
                    break;
                default:
                    break;
            }
        }
        // 添加最后一次的数据，可能没有TRANSACTIONEND
        if (!isLoopback || (enableLoopbackRemedy && needLoopback)) {
            // 添加数据解析
            for (Entry bufferEntry : transactionDataBuffer) {
                List<EventData> parseDatas = internParse(pipeline, bufferEntry);
                if (CollectionUtils.isEmpty(parseDatas)) {
                    // 可能为空，针对ddl返回时就为null
                    continue;
                }
                // 初步计算一下事件大小
                long totalSize = bufferEntry.getHeader().getEventLength();
                long eachSize = totalSize / parseDatas.size();
                for (EventData eventData : parseDatas) {
                    if (eventData == null) {
                        continue;
                    }
                    // 记录一下大小
                    eventData.setSize(eachSize);
                    if (needLoopback) {
                        // 如果延迟超过指定的阀值，则设置为需要反查db
                        if (now - eventData.getExecuteTime() > 1000 * pipeline.getParameters().getRemedyDelayThresoldForMedia()) {
                            eventData.setSyncConsistency(SyncConsistency.MEDIA);
                        } else {
                            eventData.setSyncConsistency(SyncConsistency.BASE);
                        }
                    }
                    eventDatas.add(eventData);
                }
            }
        }
    } catch (Exception e) {
        throw new SelectException(e);
    }
    return eventDatas;
}

Example 3 with RowChange

use of com.alibaba.otter.canal.protocol.CanalEntry.RowChange in project canal by alibaba.

the class BaseCanalClientTest method printEntry.

protected void printEntry(List<Entry> entrys) {
    for (Entry entry : entrys) {
        long executeTime = entry.getHeader().getExecuteTime();
        long delayTime = new Date().getTime() - executeTime;
        Date date = new Date(entry.getHeader().getExecuteTime());
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
                TransactionBegin begin = null;
                try {
                    begin = TransactionBegin.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务头信息，执行的线程id，事务耗时
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
                logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
                printXAInfo(begin.getPropsList());
            } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
                TransactionEnd end = null;
                try {
                    end = TransactionEnd.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务提交信息，事务id
                logger.info("----------------\n");
                logger.info(" END ----> transaction id: {}", end.getTransactionId());
                printXAInfo(end.getPropsList());
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
            }
            continue;
        }
        if (entry.getEntryType() == EntryType.ROWDATA) {
            RowChange rowChange = null;
            try {
                rowChange = RowChange.parseFrom(entry.getStoreValue());
            } catch (Exception e) {
                throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
            }
            EventType eventType = rowChange.getEventType();
            logger.info(row_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType, String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
            if (eventType == EventType.QUERY || rowChange.getIsDdl()) {
                logger.info("ddl : " + rowChange.getIsDdl() + " ,  sql ----> " + rowChange.getSql() + SEP);
                continue;
            }
            printXAInfo(rowChange.getPropsList());
            for (RowData rowData : rowChange.getRowDatasList()) {
                if (eventType == EventType.DELETE) {
                    printColumn(rowData.getBeforeColumnsList());
                } else if (eventType == EventType.INSERT) {
                    printColumn(rowData.getAfterColumnsList());
                } else {
                    printColumn(rowData.getAfterColumnsList());
                }
            }
        }
    }
}

Example 4 with RowChange

use of com.alibaba.otter.canal.protocol.CanalEntry.RowChange in project canal by alibaba.

the class LocalBinlogDumpTest method testSimple.

@Test
public void testSimple() {
    String directory = "/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb";
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setDirectory(directory);
    controller.setMasterPosition(startPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                    continue;
                }
                if (entry.getEntryType() == EntryType.ROWDATA) {
                    RowChange rowChange = null;
                    try {
                        rowChange = RowChange.parseFrom(entry.getStoreValue());
                    } catch (Exception e) {
                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                    }
                    EventType eventType = rowChange.getEventType();
                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                    for (RowData rowData : rowChange.getRowDatasList()) {
                        if (eventType == EventType.DELETE) {
                            print(rowData.getBeforeColumnsList());
                        } else if (eventType == EventType.INSERT) {
                            print(rowData.getAfterColumnsList());
                        } else {
                            System.out.println("-------> before");
                            print(rowData.getBeforeColumnsList());
                            System.out.println("-------> after");
                            print(rowData.getAfterColumnsList());
                        }
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 5 with RowChange

use of com.alibaba.otter.canal.protocol.CanalEntry.RowChange in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

private Entry parseRowsEvent(RowsLogEvent event) {
    if (filterRows) {
        return null;
    }
    try {
        TableMapLogEvent table = event.getTable();
        if (table == null) {
            // tableId对应的记录不存在
            throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
        }
        String fullname = table.getDbName() + "." + table.getTableName();
        // check name filter
        if (nameFilter != null && !nameFilter.filter(fullname)) {
            return null;
        }
        if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
            return null;
        }
        if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
            // 忽略rds模式的mysql.ha_health_check心跳数据
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        TableMeta tableMeta = null;
        if (tableMetaCache != null) {
            // 入错存在table meta cache
            tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
                }
            }
        }
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}