Examples with TransactionEnd com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd used on opensource projects

Example 1 with TransactionEnd

use of com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd in project canal by alibaba.

the class AbstractCanalClientTest method printEntry.

protected void printEntry(List<Entry> entrys) {
    for (Entry entry : entrys) {
        long executeTime = entry.getHeader().getExecuteTime();
        long delayTime = new Date().getTime() - executeTime;
        if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
                TransactionBegin begin = null;
                try {
                    begin = TransactionBegin.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务头信息，执行的线程id，事务耗时
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
                logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
            } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
                TransactionEnd end = null;
                try {
                    end = TransactionEnd.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务提交信息，事务id
                logger.info("----------------\n");
                logger.info(" END ----> transaction id: {}", end.getTransactionId());
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
            }
            continue;
        }
        if (entry.getEntryType() == EntryType.ROWDATA) {
            RowChange rowChage = null;
            try {
                rowChage = RowChange.parseFrom(entry.getStoreValue());
            } catch (Exception e) {
                throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
            }
            EventType eventType = rowChage.getEventType();
            logger.info(row_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType, String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
            if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
                logger.info(" sql ----> " + rowChage.getSql() + SEP);
                continue;
            }
            for (RowData rowData : rowChage.getRowDatasList()) {
                if (eventType == EventType.DELETE) {
                    printColumn(rowData.getBeforeColumnsList());
                } else if (eventType == EventType.INSERT) {
                    printColumn(rowData.getAfterColumnsList());
                } else {
                    printColumn(rowData.getAfterColumnsList());
                }
            }
        }
    }
}

Example 2 with TransactionEnd

use of com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd in project canal by alibaba.

the class BaseCanalClientTest method printEntry.

protected void printEntry(List<Entry> entrys) {
    for (Entry entry : entrys) {
        long executeTime = entry.getHeader().getExecuteTime();
        long delayTime = new Date().getTime() - executeTime;
        Date date = new Date(entry.getHeader().getExecuteTime());
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
                TransactionBegin begin = null;
                try {
                    begin = TransactionBegin.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务头信息，执行的线程id，事务耗时
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
                logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
                printXAInfo(begin.getPropsList());
            } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
                TransactionEnd end = null;
                try {
                    end = TransactionEnd.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务提交信息，事务id
                logger.info("----------------\n");
                logger.info(" END ----> transaction id: {}", end.getTransactionId());
                printXAInfo(end.getPropsList());
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
            }
            continue;
        }
        if (entry.getEntryType() == EntryType.ROWDATA) {
            RowChange rowChange = null;
            try {
                rowChange = RowChange.parseFrom(entry.getStoreValue());
            } catch (Exception e) {
                throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
            }
            EventType eventType = rowChange.getEventType();
            logger.info(row_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType, String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
            if (eventType == EventType.QUERY || rowChange.getIsDdl()) {
                logger.info("ddl : " + rowChange.getIsDdl() + " ,  sql ----> " + rowChange.getSql() + SEP);
                continue;
            }
            printXAInfo(rowChange.getPropsList());
            for (RowData rowData : rowChange.getRowDatasList()) {
                if (eventType == EventType.DELETE) {
                    printColumn(rowData.getBeforeColumnsList());
                } else if (eventType == EventType.INSERT) {
                    printColumn(rowData.getAfterColumnsList());
                } else {
                    printColumn(rowData.getAfterColumnsList());
                }
            }
        }
    }
}

Example 3 with TransactionEnd

use of com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd in project canal by alibaba.

the class LogEventConvert method parseQueryEvent.

private Entry parseQueryEvent(QueryLogEvent event) {
    String queryString = event.getQuery();
    if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
        TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
        Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
        return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
    } else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
        // MyISAM可能不会有xid事件
        TransactionEnd transactionEnd = createTransactionEnd(0L);
        Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
        return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
    } else {
        // DDL语句处理
        DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName());
        String schemaName = event.getDbName();
        if (StringUtils.isNotEmpty(result.getSchemaName())) {
            schemaName = result.getSchemaName();
        }
        String tableName = result.getTableName();
        EventType type = EventType.QUERY;
        // fixed issue https://github.com/alibaba/canal/issues/58
        if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX || result.getType() == EventType.DINDEX) {
            if (filterQueryDdl) {
                return null;
            }
            type = result.getType();
            if (StringUtils.isEmpty(tableName) || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
                // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
                throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: " + queryString + " , and DdlResult: " + result.toString());
            // return null;
            } else {
                // check name filter
                String name = schemaName + "." + tableName;
                if (nameFilter != null && !nameFilter.filter(name)) {
                    if (result.getType() == EventType.RENAME) {
                        // rename校验只要源和目标满足一个就进行操作
                        if (nameFilter != null && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                            return null;
                        }
                    } else {
                        // 其他情况返回null
                        return null;
                    }
                }
                if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
                    if (result.getType() == EventType.RENAME) {
                        // rename校验只要源和目标满足一个就进行操作
                        if (nameBlackFilter != null && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                            return null;
                        }
                    } else {
                        // 其他情况返回null
                        return null;
                    }
                }
            }
        } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE || result.getType() == EventType.DELETE) {
            // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
            if (filterQueryDml) {
                return null;
            }
        } else if (filterQueryDcl) {
            return null;
        }
        // 更新下table meta cache
        if (tableMetaCache != null && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
            for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
                // 防止rename语句后产生schema变更带来影响
                String schemaName0 = event.getDbName();
                if (StringUtils.isNotEmpty(renameResult.getSchemaName())) {
                    schemaName0 = renameResult.getSchemaName();
                }
                tableName = renameResult.getTableName();
                if (StringUtils.isNotEmpty(tableName)) {
                    // 如果解析到了正确的表信息，则根据全名进行清除
                    tableMetaCache.clearTableMeta(schemaName0, tableName);
                } else {
                    // 如果无法解析正确的表信息，则根据schema进行清除
                    tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
                }
            }
        }
        Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        if (result.getType() != EventType.QUERY) {
            rowChangeBuider.setIsDdl(true);
        }
        rowChangeBuider.setSql(queryString);
        if (StringUtils.isNotEmpty(event.getDbName())) {
            // 可能为空
            rowChangeBuider.setDdlSchemaName(event.getDbName());
        }
        rowChangeBuider.setEventType(result.getType());
        return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
    }
}

Example 4 with TransactionEnd

use of com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd in project canal by alibaba.

the class LogEventConvert method parseXidEvent.

private Entry parseXidEvent(XidLogEvent event) {
    TransactionEnd transactionEnd = createTransactionEnd(event.getXid());
    Header header = createHeader(event.getHeader(), "", "", null);
    return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
}

Example 5 with TransactionEnd

use of com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd in project canal by alibaba.

the class LogEventConvert method parseQueryEvent.

private Entry parseQueryEvent(QueryLogEvent event, boolean isSeek) {
    String queryString = event.getQuery();
    if (StringUtils.startsWithIgnoreCase(queryString, XA_START)) {
        // xa start use TransactionBegin
        TransactionBegin.Builder beginBuilder = TransactionBegin.newBuilder();
        beginBuilder.setThreadId(event.getSessionId());
        beginBuilder.addProps(createSpecialPair(XA_TYPE, XA_START));
        beginBuilder.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_START)));
        TransactionBegin transactionBegin = beginBuilder.build();
        Header header = createHeader(event.getHeader(), "", "", null);
        return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
    } else if (StringUtils.startsWithIgnoreCase(queryString, XA_END)) {
        // xa start use TransactionEnd
        TransactionEnd.Builder endBuilder = TransactionEnd.newBuilder();
        endBuilder.setTransactionId(String.valueOf(0L));
        endBuilder.addProps(createSpecialPair(XA_TYPE, XA_END));
        endBuilder.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_END)));
        TransactionEnd transactionEnd = endBuilder.build();
        Header header = createHeader(event.getHeader(), "", "", null);
        return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
    } else if (StringUtils.startsWithIgnoreCase(queryString, XA_COMMIT)) {
        // xa commit
        Header header = createHeader(event.getHeader(), "", "", EventType.XACOMMIT);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setSql(queryString);
        rowChangeBuider.addProps(createSpecialPair(XA_TYPE, XA_COMMIT));
        rowChangeBuider.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_COMMIT)));
        rowChangeBuider.setEventType(EventType.XACOMMIT);
        return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
    } else if (StringUtils.startsWithIgnoreCase(queryString, XA_ROLLBACK)) {
        // xa rollback
        Header header = createHeader(event.getHeader(), "", "", EventType.XAROLLBACK);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setSql(queryString);
        rowChangeBuider.addProps(createSpecialPair(XA_TYPE, XA_ROLLBACK));
        rowChangeBuider.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_ROLLBACK)));
        rowChangeBuider.setEventType(EventType.XAROLLBACK);
        return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
    } else if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
        TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
        Header header = createHeader(event.getHeader(), "", "", null);
        return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
    } else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
        // MyISAM可能不会有xid事件
        TransactionEnd transactionEnd = createTransactionEnd(0L);
        Header header = createHeader(event.getHeader(), "", "", null);
        return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
    } else {
        boolean notFilter = false;
        EventType type = EventType.QUERY;
        String tableName = null;
        String schemaName = null;
        if (useDruidDdlFilter) {
            List<DdlResult> results = DruidDdlParser.parse(queryString, event.getDbName());
            for (DdlResult result : results) {
                if (!processFilter(queryString, result)) {
                    // 只要有一个数据不进行过滤
                    notFilter = true;
                }
            }
            if (results.size() > 0) {
                // 如果针对多行的DDL,只能取第一条
                type = results.get(0).getType();
                schemaName = results.get(0).getSchemaName();
                tableName = results.get(0).getTableName();
            }
        } else {
            DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName());
            if (!processFilter(queryString, result)) {
                notFilter = true;
            }
            type = result.getType();
            schemaName = result.getSchemaName();
            tableName = result.getTableName();
        }
        if (!notFilter) {
            // 如果是过滤的数据就不处理了
            return null;
        }
        boolean isDml = (type == EventType.INSERT || type == EventType.UPDATE || type == EventType.DELETE);
        if (!isSeek && !isDml) {
            // 使用新的表结构元数据管理方式
            EntryPosition position = createPosition(event.getHeader());
            tableMetaCache.apply(position, event.getDbName(), queryString, null);
        }
        Header header = createHeader(event.getHeader(), schemaName, tableName, type);
        RowChange.Builder rowChangeBuilder = RowChange.newBuilder();
        rowChangeBuilder.setIsDdl(!isDml);
        rowChangeBuilder.setSql(queryString);
        if (StringUtils.isNotEmpty(event.getDbName())) {
            // 可能为空
            rowChangeBuilder.setDdlSchemaName(event.getDbName());
        }
        rowChangeBuilder.setEventType(type);
        return createEntry(header, EntryType.ROWDATA, rowChangeBuilder.build().toByteString());
    }
}