use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.
the class LogEventConvert method parseRowsQueryEvent.
private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
if (filterQueryDml) {
return null;
}
// mysql5.6支持,需要设置binlog-rows-query-log-events=1,可详细打印原始DML语句
String queryString = null;
try {
queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
String tableName = null;
if (useDruidDdlFilter) {
List<DdlResult> results = DruidDdlParser.parse(queryString, null);
if (results.size() > 0) {
tableName = results.get(0).getTableName();
}
}
return buildQueryEntry(queryString, event.getHeader(), tableName);
} catch (UnsupportedEncodingException e) {
throw new CanalParseException(e);
}
}
use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.
the class LogEventConvert method parseRowsEvent.
private Entry parseRowsEvent(RowsLogEvent event) {
if (filterRows) {
return null;
}
try {
TableMapLogEvent table = event.getTable();
if (table == null) {
// tableId对应的记录不存在
throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
}
String fullname = table.getDbName() + "." + table.getTableName();
// check name filter
if (nameFilter != null && !nameFilter.filter(fullname)) {
return null;
}
if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
return null;
}
if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
// 忽略rds模式的mysql.ha_health_check心跳数据
return null;
}
EventType eventType = null;
int type = event.getHeader().getType();
if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
eventType = EventType.INSERT;
} else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
eventType = EventType.UPDATE;
} else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
eventType = EventType.DELETE;
} else {
throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
}
Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setTableId(event.getTableId());
rowChangeBuider.setIsDdl(false);
rowChangeBuider.setEventType(eventType);
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();
boolean tableError = false;
TableMeta tableMeta = null;
if (tableMetaCache != null) {
// 入错存在table meta cache
tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
if (tableMeta == null) {
tableError = true;
if (!filterTableError) {
throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
}
}
}
while (buffer.nextOneRow(columns)) {
// 处理row记录
RowData.Builder rowDataBuilder = RowData.newBuilder();
if (EventType.INSERT == eventType) {
// insert的记录放在before字段中
tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
} else if (EventType.DELETE == eventType) {
// delete的记录放在before字段中
tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
} else {
// update需要处理before/after
tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
if (!buffer.nextOneRow(changeColumns)) {
rowChangeBuider.addRowDatas(rowDataBuilder.build());
break;
}
tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
}
rowChangeBuider.addRowDatas(rowDataBuilder.build());
}
RowChange rowChange = rowChangeBuider.build();
if (tableError) {
Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
return null;
} else {
Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
return entry;
}
} catch (Exception e) {
throw new CanalParseException("parse row data failed.", e);
}
}
use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.
the class LogEventConvert method parseQueryEvent.
private Entry parseQueryEvent(QueryLogEvent event) {
String queryString = event.getQuery();
if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
} else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
// MyISAM可能不会有xid事件
TransactionEnd transactionEnd = createTransactionEnd(0L);
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
} else {
// DDL语句处理
DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName());
String schemaName = event.getDbName();
if (StringUtils.isNotEmpty(result.getSchemaName())) {
schemaName = result.getSchemaName();
}
String tableName = result.getTableName();
EventType type = EventType.QUERY;
// fixed issue https://github.com/alibaba/canal/issues/58
if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX || result.getType() == EventType.DINDEX) {
if (filterQueryDdl) {
return null;
}
type = result.getType();
if (StringUtils.isEmpty(tableName) || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
// 如果解析不出tableName,记录一下日志,方便bugfix,目前直接抛出异常,中断解析
throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: " + queryString + " , and DdlResult: " + result.toString());
// return null;
} else {
// check name filter
String name = schemaName + "." + tableName;
if (nameFilter != null && !nameFilter.filter(name)) {
if (result.getType() == EventType.RENAME) {
// rename校验只要源和目标满足一个就进行操作
if (nameFilter != null && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
return null;
}
} else {
// 其他情况返回null
return null;
}
}
if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
if (result.getType() == EventType.RENAME) {
// rename校验只要源和目标满足一个就进行操作
if (nameBlackFilter != null && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
return null;
}
} else {
// 其他情况返回null
return null;
}
}
}
} else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE || result.getType() == EventType.DELETE) {
// 对外返回,保证兼容,还是返回QUERY类型,这里暂不解析tableName,所以无法支持过滤
if (filterQueryDml) {
return null;
}
} else if (filterQueryDcl) {
return null;
}
// 更新下table meta cache
if (tableMetaCache != null && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
// 防止rename语句后产生schema变更带来影响
String schemaName0 = event.getDbName();
if (StringUtils.isNotEmpty(renameResult.getSchemaName())) {
schemaName0 = renameResult.getSchemaName();
}
tableName = renameResult.getTableName();
if (StringUtils.isNotEmpty(tableName)) {
// 如果解析到了正确的表信息,则根据全名进行清除
tableMetaCache.clearTableMeta(schemaName0, tableName);
} else {
// 如果无法解析正确的表信息,则根据schema进行清除
tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
}
}
}
Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
if (result.getType() != EventType.QUERY) {
rowChangeBuider.setIsDdl(true);
}
rowChangeBuider.setSql(queryString);
if (StringUtils.isNotEmpty(event.getDbName())) {
// 可能为空
rowChangeBuider.setDdlSchemaName(event.getDbName());
}
rowChangeBuider.setEventType(result.getType());
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
}
use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.
the class TableMetaCache method getTableMeta.
public synchronized TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position) {
TableMeta tableMeta = null;
if (tableMetaTSDB != null) {
tableMeta = tableMetaTSDB.find(schema, table);
if (tableMeta == null) {
// 因为条件变化,可能第一次的tableMeta没取到,需要从db获取一次,并记录到snapshot中
String fullName = getFullName(schema, table);
ResultSetPacket packet = null;
String createDDL = null;
try {
try {
packet = connection.query("show create table " + fullName);
} catch (Exception e) {
// 尝试做一次retry操作
connection.reconnect();
packet = connection.query("show create table " + fullName);
}
if (packet.getFieldValues().size() > 0) {
createDDL = packet.getFieldValues().get(1);
}
// 强制覆盖掉内存值
tableMetaTSDB.apply(position, schema, createDDL, "first");
tableMeta = tableMetaTSDB.find(schema, table);
} catch (IOException e) {
throw new CanalParseException("fetch failed by table meta:" + fullName, e);
}
}
return tableMeta;
} else {
if (!useCache) {
tableMetaDB.invalidate(getFullName(schema, table));
}
return tableMetaDB.getUnchecked(getFullName(schema, table));
}
}
use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.
the class MysqlEventParser method findEndPosition.
/**
* 查询当前的binlog位置
*/
private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
try {
ResultSetPacket packet = mysqlConnection.query("show master status");
List<String> fields = packet.getFieldValues();
if (CollectionUtils.isEmpty(fields)) {
throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
}
EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
if (isGTIDMode() && fields.size() > 4) {
endPosition.setGtid(fields.get(4));
}
// MariaDB 无法通过`show master status`获取 gtid
if (mysqlConnection.isMariaDB() && isGTIDMode()) {
ResultSetPacket gtidPacket = mysqlConnection.query("SELECT @@global.gtid_binlog_pos");
List<String> gtidFields = gtidPacket.getFieldValues();
if (!CollectionUtils.isEmpty(gtidFields) && gtidFields.size() > 0) {
endPosition.setGtid(gtidFields.get(0));
}
}
return endPosition;
} catch (IOException e) {
throw new CanalParseException("command : 'show master status' has an error!", e);
}
}
Aggregations