Search in sources :

Example 1 with TableMapLogEvent

use of com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent in project canal by alibaba.

the class MysqlBinlogParsePerformanceTest method parseOneRow.

public static void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter) throws UnsupportedEncodingException {
    TableMapLogEvent map = event.getTable();
    if (map == null) {
        throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
    }
    final int columnCnt = map.getColumnCnt();
    final ColumnInfo[] columnInfo = map.getColumnInfo();
    for (int i = 0; i < columnCnt; i++) {
        if (!cols.get(i)) {
            continue;
        }
        ColumnInfo info = columnInfo[i];
        buffer.nextValue(null, i, info.type, info.meta);
        if (buffer.isNull()) {
        } else {
            buffer.getValue();
        }
    }
}
Also used : TableMapLogEvent(com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent) ColumnInfo(com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo)

Example 2 with TableMapLogEvent

use of com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent in project canal by alibaba.

the class DirectLogFetcherTest method testSimple.

@Test
public void testSimple() {
    DirectLogFetcher fetcher = new DirectLogFetcher();
    try {
        MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal");
        connector.connect();
        updateSettings(connector);
        loadBinlogChecksum(connector);
        sendRegisterSlave(connector, 3);
        sendBinlogDump(connector, "mysql-bin.000001", 4L, 3);
        fetcher.start(connector.getChannel());
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
        while (fetcher.fetch()) {
            LogEvent event = null;
            event = decoder.decode(fetcher, context);
            if (event == null) {
                throw new RuntimeException("parse failed");
            }
            int eventType = event.getHeader().getType();
            switch(eventType) {
                case LogEvent.ROTATE_EVENT:
                    // binlogFileName = ((RotateLogEvent)
                    // event).getFilename();
                    System.out.println(((RotateLogEvent) event).getFilename());
                    break;
                case LogEvent.TABLE_MAP_EVENT:
                    parseTableMapEvent((TableMapLogEvent) event);
                    break;
                case LogEvent.WRITE_ROWS_EVENT_V1:
                case LogEvent.WRITE_ROWS_EVENT:
                    parseRowsEvent((WriteRowsLogEvent) event);
                    break;
                case LogEvent.UPDATE_ROWS_EVENT_V1:
                case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
                case LogEvent.UPDATE_ROWS_EVENT:
                    parseRowsEvent((UpdateRowsLogEvent) event);
                    break;
                case LogEvent.DELETE_ROWS_EVENT_V1:
                case LogEvent.DELETE_ROWS_EVENT:
                    parseRowsEvent((DeleteRowsLogEvent) event);
                    break;
                case LogEvent.QUERY_EVENT:
                    parseQueryEvent((QueryLogEvent) event);
                    break;
                case LogEvent.ROWS_QUERY_LOG_EVENT:
                    parseRowsQueryEvent((RowsQueryLogEvent) event);
                    break;
                case LogEvent.ANNOTATE_ROWS_EVENT:
                    break;
                case LogEvent.XID_EVENT:
                    break;
                default:
                    break;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } finally {
        try {
            fetcher.close();
        } catch (IOException e) {
            Assert.fail(e.getMessage());
        }
    }
}
Also used : MysqlConnector(com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector) FormatDescriptionLogEvent(com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent) DirectLogFetcher(com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher) UpdateRowsLogEvent(com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent) LogEvent(com.taobao.tddl.dbsync.binlog.LogEvent) RowsQueryLogEvent(com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent) WriteRowsLogEvent(com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent) XidLogEvent(com.taobao.tddl.dbsync.binlog.event.XidLogEvent) TableMapLogEvent(com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent) RotateLogEvent(com.taobao.tddl.dbsync.binlog.event.RotateLogEvent) QueryLogEvent(com.taobao.tddl.dbsync.binlog.event.QueryLogEvent) DeleteRowsLogEvent(com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent) FormatDescriptionLogEvent(com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent) RowsLogEvent(com.taobao.tddl.dbsync.binlog.event.RowsLogEvent) InetSocketAddress(java.net.InetSocketAddress) LogContext(com.taobao.tddl.dbsync.binlog.LogContext) LogDecoder(com.taobao.tddl.dbsync.binlog.LogDecoder) IOException(java.io.IOException) CanalParseException(com.alibaba.otter.canal.parse.exception.CanalParseException) IOException(java.io.IOException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) Test(org.junit.Test)

Example 3 with TableMapLogEvent

use of com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent in project canal by alibaba.

the class DirectLogFetcherTest method parseOneRow.

protected void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter) throws UnsupportedEncodingException {
    TableMapLogEvent map = event.getTable();
    if (map == null) {
        throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
    }
    final int columnCnt = map.getColumnCnt();
    final ColumnInfo[] columnInfo = map.getColumnInfo();
    for (int i = 0; i < columnCnt; i++) {
        if (!cols.get(i)) {
            continue;
        }
        ColumnInfo info = columnInfo[i];
        buffer.nextValue(null, i, info.type, info.meta);
        if (buffer.isNull()) {
        // 
        } else {
            final Serializable value = buffer.getValue();
            if (value instanceof byte[]) {
                System.out.println(new String((byte[]) value));
            } else {
                System.out.println(value);
            }
        }
    }
}
Also used : TableMapLogEvent(com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent) Serializable(java.io.Serializable) ColumnInfo(com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo)

Example 4 with TableMapLogEvent

use of com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent in project canal by alibaba.

the class LogEventConvert method parseRowsEventForTableMeta.

public TableMeta parseRowsEventForTableMeta(RowsLogEvent event) {
    TableMapLogEvent table = event.getTable();
    if (table == null) {
        // tableId对应的记录不存在
        throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
    }
    boolean isHeartBeat = isAliSQLHeartBeat(table.getDbName(), table.getTableName());
    boolean isRDSHeartBeat = tableMetaCache.isOnRDS() && isRDSHeartBeat(table.getDbName(), table.getTableName());
    String fullname = table.getDbName() + "." + table.getTableName();
    // check name filter
    if (nameFilter != null && !nameFilter.filter(fullname)) {
        return null;
    }
    if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
        return null;
    }
    // if (isHeartBeat || isRDSHeartBeat) {
    // // 忽略rds模式的mysql.ha_health_check心跳数据
    // return null;
    // }
    TableMeta tableMeta = null;
    if (isRDSHeartBeat) {
        // 处理rds模式的mysql.ha_health_check心跳数据
        // 主要RDS的心跳表基本无权限,需要mock一个tableMeta
        FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0");
        FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0");
        tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
    } else if (isHeartBeat) {
        // 处理alisql模式的test.heartbeat心跳数据
        // 心跳表基本无权限,需要mock一个tableMeta
        FieldMeta idMeta = new FieldMeta("id", "smallint(6)", false, true, null);
        FieldMeta typeMeta = new FieldMeta("ts", "int(11)", true, false, null);
        tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
    }
    EntryPosition position = createPosition(event.getHeader());
    if (tableMetaCache != null && tableMeta == null) {
        // 入错存在table meta
        tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true, position);
        if (tableMeta == null) {
            if (!filterTableError) {
                throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
            }
        }
    }
    return tableMeta;
}
Also used : TableMapLogEvent(com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent) FieldMeta(com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta) TableIdNotFoundException(com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException) ByteString(com.google.protobuf.ByteString) TableMeta(com.alibaba.otter.canal.parse.inbound.TableMeta) EntryPosition(com.alibaba.otter.canal.protocol.position.EntryPosition) CanalParseException(com.alibaba.otter.canal.parse.exception.CanalParseException)

Example 5 with TableMapLogEvent

use of com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

private Entry parseRowsEvent(RowsLogEvent event) {
    if (filterRows) {
        return null;
    }
    try {
        TableMapLogEvent table = event.getTable();
        if (table == null) {
            // tableId对应的记录不存在
            throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
        }
        String fullname = table.getDbName() + "." + table.getTableName();
        // check name filter
        if (nameFilter != null && !nameFilter.filter(fullname)) {
            return null;
        }
        if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
            return null;
        }
        if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
            // 忽略rds模式的mysql.ha_health_check心跳数据
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        TableMeta tableMeta = null;
        if (tableMetaCache != null) {
            // 入错存在table meta cache
            tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
                }
            }
        }
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}
Also used : TableMapLogEvent(com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent) EventType(com.alibaba.otter.canal.protocol.CanalEntry.EventType) RowChange(com.alibaba.otter.canal.protocol.CanalEntry.RowChange) BitSet(java.util.BitSet) RowsLogBuffer(com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer) TableIdNotFoundException(com.alibaba.otter.canal.parse.exception.TableIdNotFoundException) ByteString(com.google.protobuf.ByteString) CanalParseException(com.alibaba.otter.canal.parse.exception.CanalParseException) TableIdNotFoundException(com.alibaba.otter.canal.parse.exception.TableIdNotFoundException) CanalParseException(com.alibaba.otter.canal.parse.exception.CanalParseException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) RowData(com.alibaba.otter.canal.protocol.CanalEntry.RowData) Entry(com.alibaba.otter.canal.protocol.CanalEntry.Entry) Header(com.alibaba.otter.canal.protocol.CanalEntry.Header) LogHeader(com.taobao.tddl.dbsync.binlog.event.LogHeader) TableMeta(com.alibaba.otter.canal.parse.inbound.TableMeta)

Aggregations

TableMapLogEvent (com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent)8 CanalParseException (com.alibaba.otter.canal.parse.exception.CanalParseException)4 ColumnInfo (com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo)3 UnsupportedEncodingException (java.io.UnsupportedEncodingException)3 TableMeta (com.alibaba.otter.canal.parse.inbound.TableMeta)2 Entry (com.alibaba.otter.canal.protocol.CanalEntry.Entry)2 EventType (com.alibaba.otter.canal.protocol.CanalEntry.EventType)2 Header (com.alibaba.otter.canal.protocol.CanalEntry.Header)2 RowChange (com.alibaba.otter.canal.protocol.CanalEntry.RowChange)2 RowData (com.alibaba.otter.canal.protocol.CanalEntry.RowData)2 ByteString (com.google.protobuf.ByteString)2 DeleteRowsLogEvent (com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent)2 FormatDescriptionLogEvent (com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent)2 LogHeader (com.taobao.tddl.dbsync.binlog.event.LogHeader)2 QueryLogEvent (com.taobao.tddl.dbsync.binlog.event.QueryLogEvent)2 RotateLogEvent (com.taobao.tddl.dbsync.binlog.event.RotateLogEvent)2 RowsLogBuffer (com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer)2 RowsLogEvent (com.taobao.tddl.dbsync.binlog.event.RowsLogEvent)2 RowsQueryLogEvent (com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent)2 BitSet (java.util.BitSet)2