Examples with EventType com.alibaba.otter.canal.protocol.CanalEntry.EventType used on opensource projects

Example 1 with EventType

use of com.alibaba.otter.canal.protocol.CanalEntry.EventType in project canal by alibaba.

the class AbstractCanalClientTest method printEntry.

protected void printEntry(List<Entry> entrys) {
    for (Entry entry : entrys) {
        long executeTime = entry.getHeader().getExecuteTime();
        long delayTime = new Date().getTime() - executeTime;
        if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
                TransactionBegin begin = null;
                try {
                    begin = TransactionBegin.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务头信息，执行的线程id，事务耗时
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
                logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
            } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
                TransactionEnd end = null;
                try {
                    end = TransactionEnd.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务提交信息，事务id
                logger.info("----------------\n");
                logger.info(" END ----> transaction id: {}", end.getTransactionId());
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
            }
            continue;
        }
        if (entry.getEntryType() == EntryType.ROWDATA) {
            RowChange rowChage = null;
            try {
                rowChage = RowChange.parseFrom(entry.getStoreValue());
            } catch (Exception e) {
                throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
            }
            EventType eventType = rowChage.getEventType();
            logger.info(row_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType, String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
            if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
                logger.info(" sql ----> " + rowChage.getSql() + SEP);
                continue;
            }
            for (RowData rowData : rowChage.getRowDatasList()) {
                if (eventType == EventType.DELETE) {
                    printColumn(rowData.getBeforeColumnsList());
                } else if (eventType == EventType.INSERT) {
                    printColumn(rowData.getAfterColumnsList());
                } else {
                    printColumn(rowData.getAfterColumnsList());
                }
            }
        }
    }
}

Example 2 with EventType

use of com.alibaba.otter.canal.protocol.CanalEntry.EventType in project canal by alibaba.

the class BaseCanalClientTest method printEntry.

protected void printEntry(List<Entry> entrys) {
    for (Entry entry : entrys) {
        long executeTime = entry.getHeader().getExecuteTime();
        long delayTime = new Date().getTime() - executeTime;
        Date date = new Date(entry.getHeader().getExecuteTime());
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
                TransactionBegin begin = null;
                try {
                    begin = TransactionBegin.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务头信息，执行的线程id，事务耗时
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
                logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
                printXAInfo(begin.getPropsList());
            } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
                TransactionEnd end = null;
                try {
                    end = TransactionEnd.parseFrom(entry.getStoreValue());
                } catch (InvalidProtocolBufferException e) {
                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                }
                // 打印事务提交信息，事务id
                logger.info("----------------\n");
                logger.info(" END ----> transaction id: {}", end.getTransactionId());
                printXAInfo(end.getPropsList());
                logger.info(transaction_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
            }
            continue;
        }
        if (entry.getEntryType() == EntryType.ROWDATA) {
            RowChange rowChange = null;
            try {
                rowChange = RowChange.parseFrom(entry.getStoreValue());
            } catch (Exception e) {
                throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
            }
            EventType eventType = rowChange.getEventType();
            logger.info(row_format, new Object[] { entry.getHeader().getLogfileName(), String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType, String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date), entry.getHeader().getGtid(), String.valueOf(delayTime) });
            if (eventType == EventType.QUERY || rowChange.getIsDdl()) {
                logger.info("ddl : " + rowChange.getIsDdl() + " ,  sql ----> " + rowChange.getSql() + SEP);
                continue;
            }
            printXAInfo(rowChange.getPropsList());
            for (RowData rowData : rowChange.getRowDatasList()) {
                if (eventType == EventType.DELETE) {
                    printColumn(rowData.getBeforeColumnsList());
                } else if (eventType == EventType.INSERT) {
                    printColumn(rowData.getAfterColumnsList());
                } else {
                    printColumn(rowData.getAfterColumnsList());
                }
            }
        }
    }
}

Example 3 with EventType

use of com.alibaba.otter.canal.protocol.CanalEntry.EventType in project canal by alibaba.

the class LocalBinlogDumpTest method testSimple.

@Test
public void testSimple() {
    String directory = "/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb";
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setDirectory(directory);
    controller.setMasterPosition(startPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                    continue;
                }
                if (entry.getEntryType() == EntryType.ROWDATA) {
                    RowChange rowChange = null;
                    try {
                        rowChange = RowChange.parseFrom(entry.getStoreValue());
                    } catch (Exception e) {
                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                    }
                    EventType eventType = rowChange.getEventType();
                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                    for (RowData rowData : rowChange.getRowDatasList()) {
                        if (eventType == EventType.DELETE) {
                            print(rowData.getBeforeColumnsList());
                        } else if (eventType == EventType.INSERT) {
                            print(rowData.getAfterColumnsList());
                        } else {
                            System.out.println("-------> before");
                            print(rowData.getBeforeColumnsList());
                            System.out.println("-------> after");
                            print(rowData.getAfterColumnsList());
                        }
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 4 with EventType

use of com.alibaba.otter.canal.protocol.CanalEntry.EventType in project canal by alibaba.

the class EventTransactionBuffer method add.

public void add(CanalEntry.Entry entry) throws InterruptedException {
    switch(entry.getEntryType()) {
        case TRANSACTIONBEGIN:
            // 刷新上一次的数据
            flush();
            put(entry);
            break;
        case TRANSACTIONEND:
            put(entry);
            flush();
            break;
        case ROWDATA:
            put(entry);
            // 针对非DML的数据，直接输出，不进行buffer控制
            EventType eventType = entry.getHeader().getEventType();
            if (eventType != null && !isDml(eventType)) {
                flush();
            }
            break;
        case HEARTBEAT:
            // master过来的heartbeat，说明binlog已经读完了，是idle状态
            put(entry);
            flush();
            break;
        default:
            break;
    }
}

Example 5 with EventType

use of com.alibaba.otter.canal.protocol.CanalEntry.EventType in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

private Entry parseRowsEvent(RowsLogEvent event) {
    if (filterRows) {
        return null;
    }
    try {
        TableMapLogEvent table = event.getTable();
        if (table == null) {
            // tableId对应的记录不存在
            throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
        }
        String fullname = table.getDbName() + "." + table.getTableName();
        // check name filter
        if (nameFilter != null && !nameFilter.filter(fullname)) {
            return null;
        }
        if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
            return null;
        }
        if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
            // 忽略rds模式的mysql.ha_health_check心跳数据
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        TableMeta tableMeta = null;
        if (tableMetaCache != null) {
            // 入错存在table meta cache
            tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
                }
            }
        }
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}