Examples with TableMeta com.alibaba.otter.canal.parse.inbound.TableMeta used on opensource projects

Example 6 with TableMeta

use of com.alibaba.otter.canal.parse.inbound.TableMeta in project canal by alibaba.

the class DatabaseTableMeta method compareTableMetaDbAndMemory.

private boolean compareTableMetaDbAndMemory(MysqlConnection connection, MemoryTableMeta memoryTableMeta, final String schema, final String table) {
    TableMeta tableMetaFromMem = memoryTableMeta.find(schema, table);
    TableMeta tableMetaFromDB = new TableMeta();
    tableMetaFromDB.setSchema(schema);
    tableMetaFromDB.setTable(table);
    String createDDL = null;
    try {
        ResultSetPacket packet = connection.query("show create table " + getFullName(schema, table));
        if (packet.getFieldValues().size() > 1) {
            createDDL = packet.getFieldValues().get(1);
            tableMetaFromDB.setFields(TableMetaCache.parseTableMeta(schema, table, packet));
        }
    } catch (Throwable e) {
        try {
            // retry for broke pipe, see:
            // https://github.com/alibaba/canal/issues/724
            connection.reconnect();
            ResultSetPacket packet = connection.query("show create table " + getFullName(schema, table));
            if (packet.getFieldValues().size() > 1) {
                createDDL = packet.getFieldValues().get(1);
                tableMetaFromDB.setFields(TableMetaCache.parseTableMeta(schema, table, packet));
            }
        } catch (IOException e1) {
            if (e.getMessage().contains("errorNumber=1146")) {
                logger.error("table not exist in db , pls check :" + getFullName(schema, table) + " , mem : " + tableMetaFromMem);
                return false;
            }
            throw new CanalParseException(e);
        }
    }
    boolean result = compareTableMeta(tableMetaFromMem, tableMetaFromDB);
    if (!result) {
        logger.error("pls submit github issue, show create table ddl:" + createDDL + " , compare failed . \n db : " + tableMetaFromDB + " \n mem : " + tableMetaFromMem);
    }
    return result;
}

Example 7 with TableMeta

use of com.alibaba.otter.canal.parse.inbound.TableMeta in project canal by alibaba.

the class LogEventConvert method parseRowsEventForTableMeta.

public TableMeta parseRowsEventForTableMeta(RowsLogEvent event) {
    TableMapLogEvent table = event.getTable();
    if (table == null) {
        // tableId对应的记录不存在
        throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
    }
    boolean isHeartBeat = isAliSQLHeartBeat(table.getDbName(), table.getTableName());
    boolean isRDSHeartBeat = tableMetaCache.isOnRDS() && isRDSHeartBeat(table.getDbName(), table.getTableName());
    String fullname = table.getDbName() + "." + table.getTableName();
    // check name filter
    if (nameFilter != null && !nameFilter.filter(fullname)) {
        return null;
    }
    if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
        return null;
    }
    // if (isHeartBeat || isRDSHeartBeat) {
    // // 忽略rds模式的mysql.ha_health_check心跳数据
    // return null;
    // }
    TableMeta tableMeta = null;
    if (isRDSHeartBeat) {
        // 处理rds模式的mysql.ha_health_check心跳数据
        // 主要RDS的心跳表基本无权限,需要mock一个tableMeta
        FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0");
        FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0");
        tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
    } else if (isHeartBeat) {
        // 处理alisql模式的test.heartbeat心跳数据
        // 心跳表基本无权限,需要mock一个tableMeta
        FieldMeta idMeta = new FieldMeta("id", "smallint(6)", false, true, null);
        FieldMeta typeMeta = new FieldMeta("ts", "int(11)", true, false, null);
        tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
    }
    EntryPosition position = createPosition(event.getHeader());
    if (tableMetaCache != null && tableMeta == null) {
        // 入错存在table meta
        tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true, position);
        if (tableMeta == null) {
            if (!filterTableError) {
                throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
            }
        }
    }
    return tableMeta;
}

Example 8 with TableMeta

use of com.alibaba.otter.canal.parse.inbound.TableMeta in project canal by alibaba.

the class MemoryTableMeta method parse.

private TableMeta parse(SQLCreateTableStatement statement) {
    int size = statement.getTableElementList().size();
    if (size > 0) {
        TableMeta tableMeta = new TableMeta();
        for (int i = 0; i < size; ++i) {
            SQLTableElement element = statement.getTableElementList().get(i);
            processTableElement(element, tableMeta);
        }
        return tableMeta;
    }
    return null;
}

Example 9 with TableMeta

use of com.alibaba.otter.canal.parse.inbound.TableMeta in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

private Entry parseRowsEvent(RowsLogEvent event) {
    if (filterRows) {
        return null;
    }
    try {
        TableMapLogEvent table = event.getTable();
        if (table == null) {
            // tableId对应的记录不存在
            throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
        }
        String fullname = table.getDbName() + "." + table.getTableName();
        // check name filter
        if (nameFilter != null && !nameFilter.filter(fullname)) {
            return null;
        }
        if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
            return null;
        }
        if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
            // 忽略rds模式的mysql.ha_health_check心跳数据
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        TableMeta tableMeta = null;
        if (tableMetaCache != null) {
            // 入错存在table meta cache
            tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
                }
            }
        }
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}

Example 10 with TableMeta

use of com.alibaba.otter.canal.parse.inbound.TableMeta in project canal by alibaba.

the class TableMetaCache method getTableMeta.

public synchronized TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position) {
    TableMeta tableMeta = null;
    if (tableMetaTSDB != null) {
        tableMeta = tableMetaTSDB.find(schema, table);
        if (tableMeta == null) {
            // 因为条件变化，可能第一次的tableMeta没取到，需要从db获取一次，并记录到snapshot中
            String fullName = getFullName(schema, table);
            ResultSetPacket packet = null;
            String createDDL = null;
            try {
                try {
                    packet = connection.query("show create table " + fullName);
                } catch (Exception e) {
                    // 尝试做一次retry操作
                    connection.reconnect();
                    packet = connection.query("show create table " + fullName);
                }
                if (packet.getFieldValues().size() > 0) {
                    createDDL = packet.getFieldValues().get(1);
                }
                // 强制覆盖掉内存值
                tableMetaTSDB.apply(position, schema, createDDL, "first");
                tableMeta = tableMetaTSDB.find(schema, table);
            } catch (IOException e) {
                throw new CanalParseException("fetch failed by table meta:" + fullName, e);
            }
        }
        return tableMeta;
    } else {
        if (!useCache) {
            tableMetaDB.invalidate(getFullName(schema, table));
        }
        return tableMetaDB.getUnchecked(getFullName(schema, table));
    }
}