Examples with RowsLogBuffer com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer used on opensource projects

Example 1 with RowsLogBuffer

use of com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer in project canal by alibaba.

the class DirectLogFetcherTest method parseRowsEvent.

protected void parseRowsEvent(RowsLogEvent event) {
    try {
        System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName, event.getHeader().getLogPos() - event.getHeader().getEventLen(), event.getTable().getDbName(), event.getTable().getTableName()));
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            int type = event.getHeader().getType();
            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                // insert的记录放在before字段中
                parseOneRow(event, buffer, columns, true);
            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                // delete的记录放在before字段中
                parseOneRow(event, buffer, columns, false);
            } else {
                // update需要处理before/after
                System.out.println("-------> before");
                parseOneRow(event, buffer, columns, false);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    break;
                }
                System.out.println("-------> after");
                parseOneRow(event, buffer, changeColumns, true);
            }
        }
    } catch (Exception e) {
        throw new RuntimeException("parse row data failed.", e);
    }
}

Example 2 with RowsLogBuffer

use of com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

private Entry parseRowsEvent(RowsLogEvent event) {
    if (filterRows) {
        return null;
    }
    try {
        TableMapLogEvent table = event.getTable();
        if (table == null) {
            // tableId对应的记录不存在
            throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
        }
        String fullname = table.getDbName() + "." + table.getTableName();
        // check name filter
        if (nameFilter != null && !nameFilter.filter(fullname)) {
            return null;
        }
        if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
            return null;
        }
        if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
            // 忽略rds模式的mysql.ha_health_check心跳数据
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        TableMeta tableMeta = null;
        if (tableMetaCache != null) {
            // 入错存在table meta cache
            tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
                }
            }
        }
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}

Example 3 with RowsLogBuffer

use of com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer in project canal by alibaba.

the class BaseLogFetcherTest method parseRowsEvent.

protected void parseRowsEvent(RowsLogEvent event) {
    try {
        System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName, event.getHeader().getLogPos() - event.getHeader().getEventLen(), event.getTable().getDbName(), event.getTable().getTableName()));
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            int type = event.getHeader().getType();
            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                // insert的记录放在before字段中
                parseOneRow(event, buffer, columns, true);
            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                // delete的记录放在before字段中
                parseOneRow(event, buffer, columns, false);
            } else {
                // update需要处理before/after
                System.out.println("-------> before");
                parseOneRow(event, buffer, columns, false);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    break;
                }
                System.out.println("-------> after");
                parseOneRow(event, buffer, changeColumns, true);
            }
        }
    } catch (Exception e) {
        throw new RuntimeException("parse row data failed.", e);
    }
}

Example 4 with RowsLogBuffer

use of com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

public Entry parseRowsEvent(RowsLogEvent event, TableMeta tableMeta) {
    if (filterRows) {
        return null;
    }
    try {
        if (tableMeta == null) {
            // 如果没有外部指定
            tableMeta = parseRowsEventForTableMeta(event);
        }
        if (tableMeta == null) {
            // 拿不到表结构,执行忽略
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type || LogEvent.PARTIAL_UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        int rowsCount = 0;
        while (buffer.nextOneRow(columns, false)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowsCount++;
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        TableMapLogEvent table = event.getTable();
        Header header = createHeader(event.getHeader(), table.getDbName(), table.getTableName(), eventType, rowsCount);
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChange.toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}

Example 5 with RowsLogBuffer

use of com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer in project canal by alibaba.

the class MysqlBinlogParsePerformanceTest method parseRowsEvent.

public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
    try {
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        while (buffer.nextOneRow(columns)) {
            int type = event.getHeader().getType();
            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                parseOneRow(event, buffer, columns, true);
            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                parseOneRow(event, buffer, columns, false);
            } else {
                parseOneRow(event, buffer, columns, false);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    break;
                }
                parseOneRow(event, buffer, changeColumns, true);
            }
            sum.incrementAndGet();
        }
    } catch (Exception e) {
        throw new RuntimeException("parse row data failed.", e);
    }
}