use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.
the class LogEventConvert method parseQueryEvent.
private Entry parseQueryEvent(QueryLogEvent event) {
String queryString = event.getQuery();
if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
} else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
// MyISAM可能不会有xid事件
TransactionEnd transactionEnd = createTransactionEnd(0L);
Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
} else {
// DDL语句处理
DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName());
String schemaName = event.getDbName();
if (StringUtils.isNotEmpty(result.getSchemaName())) {
schemaName = result.getSchemaName();
}
String tableName = result.getTableName();
EventType type = EventType.QUERY;
// fixed issue https://github.com/alibaba/canal/issues/58
if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX || result.getType() == EventType.DINDEX) {
if (filterQueryDdl) {
return null;
}
type = result.getType();
if (StringUtils.isEmpty(tableName) || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
// 如果解析不出tableName,记录一下日志,方便bugfix,目前直接抛出异常,中断解析
throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: " + queryString + " , and DdlResult: " + result.toString());
// return null;
} else {
// check name filter
String name = schemaName + "." + tableName;
if (nameFilter != null && !nameFilter.filter(name)) {
if (result.getType() == EventType.RENAME) {
// rename校验只要源和目标满足一个就进行操作
if (nameFilter != null && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
return null;
}
} else {
// 其他情况返回null
return null;
}
}
if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
if (result.getType() == EventType.RENAME) {
// rename校验只要源和目标满足一个就进行操作
if (nameBlackFilter != null && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
return null;
}
} else {
// 其他情况返回null
return null;
}
}
}
} else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE || result.getType() == EventType.DELETE) {
// 对外返回,保证兼容,还是返回QUERY类型,这里暂不解析tableName,所以无法支持过滤
if (filterQueryDml) {
return null;
}
} else if (filterQueryDcl) {
return null;
}
// 更新下table meta cache
if (tableMetaCache != null && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
// 防止rename语句后产生schema变更带来影响
String schemaName0 = event.getDbName();
if (StringUtils.isNotEmpty(renameResult.getSchemaName())) {
schemaName0 = renameResult.getSchemaName();
}
tableName = renameResult.getTableName();
if (StringUtils.isNotEmpty(tableName)) {
// 如果解析到了正确的表信息,则根据全名进行清除
tableMetaCache.clearTableMeta(schemaName0, tableName);
} else {
// 如果无法解析正确的表信息,则根据schema进行清除
tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
}
}
}
Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
if (result.getType() != EventType.QUERY) {
rowChangeBuider.setIsDdl(true);
}
rowChangeBuider.setSql(queryString);
if (StringUtils.isNotEmpty(event.getDbName())) {
// 可能为空
rowChangeBuider.setDdlSchemaName(event.getDbName());
}
rowChangeBuider.setEventType(result.getType());
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
}
use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.
the class LogEventConvert method parseGTIDLogEvent.
private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
LogHeader logHeader = logEvent.getHeader();
Pair.Builder builder = Pair.newBuilder();
builder.setKey("gtid");
builder.setValue(logEvent.getGtidStr());
if (logEvent.getLastCommitted() != null) {
builder.setKey("lastCommitted");
builder.setValue(String.valueOf(logEvent.getLastCommitted()));
builder.setKey("sequenceNumber");
builder.setValue(String.valueOf(logEvent.getSequenceNumber()));
}
Header header = createHeader(logHeader, "", "", EventType.GTID);
return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}
use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.
the class LogEventConvert method parseRowsEvent.
public Entry parseRowsEvent(RowsLogEvent event, TableMeta tableMeta) {
if (filterRows) {
return null;
}
try {
if (tableMeta == null) {
// 如果没有外部指定
tableMeta = parseRowsEventForTableMeta(event);
}
if (tableMeta == null) {
// 拿不到表结构,执行忽略
return null;
}
EventType eventType = null;
int type = event.getHeader().getType();
if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
eventType = EventType.INSERT;
} else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type || LogEvent.PARTIAL_UPDATE_ROWS_EVENT == type) {
eventType = EventType.UPDATE;
} else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
eventType = EventType.DELETE;
} else {
throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
}
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setTableId(event.getTableId());
rowChangeBuider.setIsDdl(false);
rowChangeBuider.setEventType(eventType);
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();
boolean tableError = false;
int rowsCount = 0;
while (buffer.nextOneRow(columns, false)) {
// 处理row记录
RowData.Builder rowDataBuilder = RowData.newBuilder();
if (EventType.INSERT == eventType) {
// insert的记录放在before字段中
tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
} else if (EventType.DELETE == eventType) {
// delete的记录放在before字段中
tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
} else {
// update需要处理before/after
tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
if (!buffer.nextOneRow(changeColumns, true)) {
rowChangeBuider.addRowDatas(rowDataBuilder.build());
break;
}
tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
}
rowsCount++;
rowChangeBuider.addRowDatas(rowDataBuilder.build());
}
TableMapLogEvent table = event.getTable();
Header header = createHeader(event.getHeader(), table.getDbName(), table.getTableName(), eventType, rowsCount);
RowChange rowChange = rowChangeBuider.build();
if (tableError) {
Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
return null;
} else {
Entry entry = createEntry(header, EntryType.ROWDATA, rowChange.toByteString());
return entry;
}
} catch (Exception e) {
throw new CanalParseException("parse row data failed.", e);
}
}
use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.
the class LogEventConvert method parseXidEvent.
private Entry parseXidEvent(XidLogEvent event) {
TransactionEnd transactionEnd = createTransactionEnd(event.getXid());
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
}
use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.
the class LogEventConvert method parseQueryEvent.
private Entry parseQueryEvent(QueryLogEvent event, boolean isSeek) {
String queryString = event.getQuery();
if (StringUtils.startsWithIgnoreCase(queryString, XA_START)) {
// xa start use TransactionBegin
TransactionBegin.Builder beginBuilder = TransactionBegin.newBuilder();
beginBuilder.setThreadId(event.getSessionId());
beginBuilder.addProps(createSpecialPair(XA_TYPE, XA_START));
beginBuilder.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_START)));
TransactionBegin transactionBegin = beginBuilder.build();
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
} else if (StringUtils.startsWithIgnoreCase(queryString, XA_END)) {
// xa start use TransactionEnd
TransactionEnd.Builder endBuilder = TransactionEnd.newBuilder();
endBuilder.setTransactionId(String.valueOf(0L));
endBuilder.addProps(createSpecialPair(XA_TYPE, XA_END));
endBuilder.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_END)));
TransactionEnd transactionEnd = endBuilder.build();
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
} else if (StringUtils.startsWithIgnoreCase(queryString, XA_COMMIT)) {
// xa commit
Header header = createHeader(event.getHeader(), "", "", EventType.XACOMMIT);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setSql(queryString);
rowChangeBuider.addProps(createSpecialPair(XA_TYPE, XA_COMMIT));
rowChangeBuider.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_COMMIT)));
rowChangeBuider.setEventType(EventType.XACOMMIT);
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
} else if (StringUtils.startsWithIgnoreCase(queryString, XA_ROLLBACK)) {
// xa rollback
Header header = createHeader(event.getHeader(), "", "", EventType.XAROLLBACK);
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setSql(queryString);
rowChangeBuider.addProps(createSpecialPair(XA_TYPE, XA_ROLLBACK));
rowChangeBuider.addProps(createSpecialPair(XA_XID, getXaXid(queryString, XA_ROLLBACK)));
rowChangeBuider.setEventType(EventType.XAROLLBACK);
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
} else if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
} else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
// MyISAM可能不会有xid事件
TransactionEnd transactionEnd = createTransactionEnd(0L);
Header header = createHeader(event.getHeader(), "", "", null);
return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
} else {
boolean notFilter = false;
EventType type = EventType.QUERY;
String tableName = null;
String schemaName = null;
if (useDruidDdlFilter) {
List<DdlResult> results = DruidDdlParser.parse(queryString, event.getDbName());
for (DdlResult result : results) {
if (!processFilter(queryString, result)) {
// 只要有一个数据不进行过滤
notFilter = true;
}
}
if (results.size() > 0) {
// 如果针对多行的DDL,只能取第一条
type = results.get(0).getType();
schemaName = results.get(0).getSchemaName();
tableName = results.get(0).getTableName();
}
} else {
DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName());
if (!processFilter(queryString, result)) {
notFilter = true;
}
type = result.getType();
schemaName = result.getSchemaName();
tableName = result.getTableName();
}
if (!notFilter) {
// 如果是过滤的数据就不处理了
return null;
}
boolean isDml = (type == EventType.INSERT || type == EventType.UPDATE || type == EventType.DELETE);
if (!isSeek && !isDml) {
// 使用新的表结构元数据管理方式
EntryPosition position = createPosition(event.getHeader());
tableMetaCache.apply(position, event.getDbName(), queryString, null);
}
Header header = createHeader(event.getHeader(), schemaName, tableName, type);
RowChange.Builder rowChangeBuilder = RowChange.newBuilder();
rowChangeBuilder.setIsDdl(!isDml);
rowChangeBuilder.setSql(queryString);
if (StringUtils.isNotEmpty(event.getDbName())) {
// 可能为空
rowChangeBuilder.setDdlSchemaName(event.getDbName());
}
rowChangeBuilder.setEventType(type);
return createEntry(header, EntryType.ROWDATA, rowChangeBuilder.build().toByteString());
}
}
Aggregations