use of com.alibaba.otter.shared.etl.model.EventType in project otter by alibaba.
the class MessageParser method internParse.
/**
* 解析出从canal中获取的Event事件<br>
* Oracle:有变更的列值. <br>
* <i>insert:从afterColumns中获取所有的变更数据<br>
* <i>delete:从beforeColumns中获取所有的变更数据<br>
* <i>update:在before中存放所有的主键和变化前的非主键值,在after中存放变化后的主键和非主键值,如果是复合主键,只会存放变化的主键<br>
* Mysql:可以得到所有变更前和变更后的数据.<br>
* <i>insert:从afterColumns中获取所有的变更数据<br>
* <i>delete:从beforeColumns中获取所有的变更数据<br>
* <i>update:在beforeColumns中存放变更前的所有数据,在afterColumns中存放变更后的所有数据<br>
*/
private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChange, RowData rowData) {
EventData eventData = new EventData();
eventData.setTableName(entry.getHeader().getTableName());
eventData.setSchemaName(entry.getHeader().getSchemaName());
eventData.setEventType(EventType.valueOf(rowChange.getEventType().name()));
eventData.setExecuteTime(entry.getHeader().getExecuteTime());
EventType eventType = eventData.getEventType();
TableInfoHolder tableHolder = null;
if (!StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), eventData.getSchemaName())) {
boolean useTableTransform = pipeline.getParameters().getUseTableTransform();
Table table = null;
DataMediaPair dataMediaPair = ConfigHelper.findDataMediaPairBySourceName(pipeline, eventData.getSchemaName(), eventData.getTableName());
DataMedia dataMedia = dataMediaPair.getSource();
eventData.setTableId(dataMedia.getId());
// 获取目标表
DataMedia targetDataMedia = dataMediaPair.getTarget();
if (useTableTransform || dataMedia.getSource().getType().isOracle()) {
// oracle需要反查一次meta
// 如果设置了需要进行table meta转化,则反查一下table信息
// 比如oracle erosa解析时可能使用了非物理主键,需要直接使用,信任erosa的信息
DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
// 查询一下meta信息
table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
if (table == null) {
logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(), eventData.getTableName());
}
// 获取一下目标库的拆分字段,设置源表为主键
// 首先要求源和目标的库名表名是一致的
DbDialect targetDbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) targetDataMedia.getSource());
if (targetDbDialect.isDRDS()) {
String schemaName = buildName(eventData.getSchemaName(), dataMedia.getNamespaceMode(), targetDataMedia.getNamespaceMode());
String tableName = buildName(eventData.getSchemaName(), dataMedia.getNameMode(), targetDataMedia.getNameMode());
String shardColumns = targetDbDialect.getShardColumns(schemaName, tableName);
if (StringUtils.isNotEmpty(shardColumns)) {
String[] columns = StringUtils.split(shardColumns, ',');
for (String key : columns) {
org.apache.ddlutils.model.Column col = table.findColumn(key, false);
if (col != null) {
col.setPrimaryKey(true);
} else {
logger.warn(String.format("shardColumn %s in table[%s.%s] is not found", key, eventData.getSchemaName(), eventData.getTableName()));
}
}
}
}
tableHolder = new TableInfoHolder(dbDialect, table, useTableTransform);
}
}
List<Column> beforeColumns = rowData.getBeforeColumnsList();
List<Column> afterColumns = rowData.getAfterColumnsList();
String tableName = eventData.getSchemaName() + "." + eventData.getTableName();
// 判断一下是否需要all columns
// 如果是rowMode模式,所有字段都需要标记为updated
boolean isRowMode = pipeline.getParameters().getSyncMode().isRow();
boolean needAllColumns = isRowMode || checkNeedAllColumns(pipeline);
// 变更后的主键
Map<String, EventColumn> keyColumns = new LinkedHashMap<String, EventColumn>();
// 变更前的主键
Map<String, EventColumn> oldKeyColumns = new LinkedHashMap<String, EventColumn>();
// 有变化的非主键
Map<String, EventColumn> notKeyColumns = new LinkedHashMap<String, EventColumn>();
if (eventType.isInsert()) {
for (Column column : afterColumns) {
if (isKey(tableHolder, tableName, column)) {
keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
} else {
// mysql 有效
notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
}
}
} else if (eventType.isDelete()) {
for (Column column : beforeColumns) {
if (isKey(tableHolder, tableName, column)) {
keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
} else {
// mysql 有效
notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
}
}
} else if (eventType.isUpdate()) {
// 获取变更前的主键.
for (Column column : beforeColumns) {
if (isKey(tableHolder, tableName, column)) {
oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
// 同时记录一下new
// key,因为mysql5.6之后出现了minimal模式,after里会没有主键信息,需要在before记录中找
keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
} else {
if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
// 针对行记录同步时,针对oracle记录一下非主键的字段,因为update时针对未变更的字段在aftercolume里没有
notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode, tableHolder));
}
}
}
for (Column column : afterColumns) {
if (isKey(tableHolder, tableName, column)) {
// 获取变更后的主键
keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
} else if (needAllColumns || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE || column.getUpdated()) {
// 在update操作时,oracle和mysql存放变更的非主键值的方式不同,oracle只有变更的字段;
// mysql会把变更前和变更后的字段都发出来,只需要取有变更的字段.
// 如果是oracle库,after里一定为对应的变更字段
boolean isUpdate = true;
if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) {
// mysql的after里部分数据为未变更,oracle里after里为变更字段
isUpdate = column.getUpdated();
}
// 如果是rowMode,所有字段都为updated
notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode || isUpdate, tableHolder));
}
}
if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
// 针对oracle进行特殊处理
checkUpdateKeyColumns(oldKeyColumns, keyColumns);
}
}
List<EventColumn> keys = new ArrayList<EventColumn>(keyColumns.values());
List<EventColumn> oldKeys = new ArrayList<EventColumn>(oldKeyColumns.values());
List<EventColumn> columns = new ArrayList<EventColumn>(notKeyColumns.values());
Collections.sort(keys, new EventColumnIndexComparable());
Collections.sort(oldKeys, new EventColumnIndexComparable());
Collections.sort(columns, new EventColumnIndexComparable());
if (!keyColumns.isEmpty()) {
eventData.setKeys(keys);
if (eventData.getEventType().isUpdate() && !oldKeys.equals(keys)) {
// update类型,如果存在主键不同,则记录下old
// keys为变更前的主键
eventData.setOldKeys(oldKeys);
}
eventData.setColumns(columns);
// } else if (CanalEntry.Type.MYSQL ==
// entry.getHeader().getSourceType()) {
// // 只支持mysql无主键同步
// if (eventType.isUpdate()) {
// List<EventColumn> oldColumns = new ArrayList<EventColumn>();
// List<EventColumn> newColumns = new ArrayList<EventColumn>();
// for (Column column : beforeColumns) {
// oldColumns.add(copyEventColumn(column, true, tableHolder));
// }
//
// for (Column column : afterColumns) {
// newColumns.add(copyEventColumn(column, true, tableHolder));
// }
// Collections.sort(oldColumns, new EventColumnIndexComparable());
// Collections.sort(newColumns, new EventColumnIndexComparable());
// eventData.setOldKeys(oldColumns);// 做为老主键
// eventData.setKeys(newColumns);// 做为新主键,需要保证新老主键字段数量一致
// } else {
// // 针对无主键,等同为所有都是主键进行处理
// eventData.setKeys(columns);
// }
} else {
throw new SelectException("this rowdata has no pks , entry: " + entry.toString() + " and rowData: " + rowData);
}
return eventData;
}
use of com.alibaba.otter.shared.etl.model.EventType in project otter by alibaba.
the class MessageParser method internParse.
private List<EventData> internParse(Pipeline pipeline, Entry entry) {
RowChange rowChange = null;
try {
rowChange = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new SelectException("parser of canal-event has an error , data:" + entry.toString(), e);
}
if (rowChange == null) {
return null;
}
String schemaName = entry.getHeader().getSchemaName();
String tableName = entry.getHeader().getTableName();
EventType eventType = EventType.valueOf(rowChange.getEventType().name());
// 处理下DDL操作
if (eventType.isQuery()) {
// 直接忽略query事件
return null;
}
// 首先判断是否为系统表
if (StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), schemaName)) {
// do noting
if (eventType.isDdl()) {
return null;
}
if (StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemDualTable(), tableName)) {
// 心跳表数据直接忽略
return null;
}
} else {
if (eventType.isDdl()) {
boolean notExistReturnNull = false;
if (eventType.isRename()) {
notExistReturnNull = true;
}
DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, schemaName, tableName, notExistReturnNull);
// DataMediaInfo;并且把CREATE/ALTER类型的事件丢弃掉.
if (dataMedia != null && (eventType.isCreate() || eventType.isAlter() || eventType.isRename())) {
DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
// 更新下meta信息
dbDialect.reloadTable(schemaName, tableName);
}
boolean ddlSync = pipeline.getParameters().getDdlSync();
if (ddlSync) {
// 处理下ddl操作
EventData eventData = new EventData();
eventData.setSchemaName(schemaName);
eventData.setTableName(tableName);
eventData.setEventType(eventType);
eventData.setExecuteTime(entry.getHeader().getExecuteTime());
eventData.setSql(rowChange.getSql());
eventData.setDdlSchemaName(rowChange.getDdlSchemaName());
eventData.setTableId(dataMedia.getId());
return Arrays.asList(eventData);
} else {
return null;
}
}
}
List<EventData> eventDatas = new ArrayList<EventData>();
for (RowData rowData : rowChange.getRowDatasList()) {
EventData eventData = internParse(pipeline, entry, rowChange, rowData);
if (eventData != null) {
eventDatas.add(eventData);
}
}
return eventDatas;
}
use of com.alibaba.otter.shared.etl.model.EventType in project otter by alibaba.
the class RowDataTransformer method translateColumn.
private EventColumn translateColumn(EventData data, EventColumn scolumn, TableInfoHolder tableHolder, DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames) {
EventType type = data.getEventType();
EventColumn tcolumn = new EventColumn();
tcolumn.setNull(scolumn.getColumnValue() == null);
// 左右两边的主键值必须保持一样,可以不为物理主键
tcolumn.setKey(scolumn.isKey());
tcolumn.setIndex(scolumn.getIndex());
tcolumn.setUpdate(scolumn.isUpdate());
String columnName = translateColumnName(scolumn.getColumnName(), dataMediaPair, translateColumnNames);
if (StringUtils.isBlank(columnName)) {
throw new TransformException("can't translate column name:" + scolumn.getColumnName() + "in pair:" + dataMediaPair.toString());
}
// 特殊处理
// columnName = StringUtils.remove(columnName, "`"); //
// 处理下特殊字符,eromanga给了错误的字段名
tcolumn.setColumnName(columnName);
// 不反查,直接使用源库的类型
tcolumn.setColumnType(scolumn.getColumnType());
if (tableHolder != null) {
// modify by ljh at 2013-01-23
// 双向同步新增字段,在一边加了字段后,虽然新的字段没有产生业务变化,但会因为某些原因导致传递了新的字段到T模块
// 此时在目标库并不存在这个字段,导致一直挂起。ps. mysql新增字段时间不是一般的长
// 所以,做了一个容错处理,针对目标库不存在的字段,如果变更记录在源库不存在变更,并且是null值的,允许丢弃该字段(其实最好还是要判断源库的column的defaultValue和当前值是否一致)
boolean canColumnsNotExist = tableHolder.isEnableCompatibleMissColumn();
if (type == EventType.UPDATE) {
// 非变更字段,且当前值为null
canColumnsNotExist &= !scolumn.isUpdate() && scolumn.isNull();
} else if (type == EventType.INSERT) {
// 当前值为null
canColumnsNotExist &= scolumn.isNull();
} else if (type == EventType.DELETE) {
// 主键不允许不存在
canColumnsNotExist &= !scolumn.isKey();
}
Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
// 匹配字段为空,可能源库发生过DDL操作,目标库重新载入一下meta信息
if (matchDbColumn == null) {
// 尝试reload一下table meta
// 获取目标库的表信息
DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource();
DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource);
String schemaName = tableHolder.getTable().getSchema();
if (StringUtils.isEmpty(schemaName)) {
schemaName = tableHolder.getTable().getCatalog();
}
// 强制反查一次,并放入cache
Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false);
tableHolder.setTable(table);
matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
if (matchDbColumn == null) {
if (canColumnsNotExist) {
return null;
} else {
throw new TransformException(scolumn.getColumnName() + " is not found in " + table.toString() + " and source : " + dataMediaPair.getTarget().getNamespace() + "." + dataMediaPair.getTarget().getName());
}
}
}
if (tableHolder.isUseTableTransform()) {
int sqlType = matchDbColumn.getTypeCode();
tcolumn.setColumnType(sqlType);
}
}
// if (dataMediaPair.getTarget().getSource().getType().isOracle()) {
// // 特殊处理下oracle编码
// String encodeValue = SqlUtils.encoding(scolumn.getColumnValue(),
// scolumn.getColumnType(),
// dataMediaPair.getSource().getSource().getEncode(),
// dataMediaPair.getTarget().getSource().getEncode());
// tcolumn.setColumnValue(encodeValue);
// } else {
// mysql编码转化已经在驱动层面上完成
tcolumn.setColumnValue(scolumn.getColumnValue());
// }
// 删除映射关系,避免下次重复转换
translateColumnNames.remove(scolumn.getColumnName(), columnName);
return tcolumn;
}
Aggregations