Examples with EventColumn com.alibaba.otter.shared.etl.model.EventColumn used on opensource projects

Example 1 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DbLoadMerger method replaceColumnValue.

/**
 * 把old中的值存在而new中不存在的值合并到new中,并且把old中的变更前的主键保存到new中的变更前的主键.
 *
 * @param newEventData
 * @param oldEventData
 * @return
 */
private static EventData replaceColumnValue(EventData newEventData, EventData oldEventData) {
    List<EventColumn> newColumns = newEventData.getColumns();
    List<EventColumn> oldColumns = oldEventData.getColumns();
    List<EventColumn> temp = new ArrayList<EventColumn>();
    for (EventColumn oldColumn : oldColumns) {
        boolean contain = false;
        for (EventColumn newColumn : newColumns) {
            if (oldColumn.getColumnName().equalsIgnoreCase(newColumn.getColumnName())) {
                // 合并isUpdate字段
                newColumn.setUpdate(newColumn.isUpdate() || oldColumn.isUpdate());
                contain = true;
            }
        }
        if (!contain) {
            temp.add(oldColumn);
        }
    }
    newColumns.addAll(temp);
    // 排序
    Collections.sort(newColumns, new EventColumnIndexComparable());
    // 把上一次变更的旧主键传递到这次变更的旧主键.
    newEventData.setOldKeys(oldEventData.getOldKeys());
    if (oldEventData.getSyncConsistency() != null) {
        newEventData.setSyncConsistency(oldEventData.getSyncConsistency());
    }
    if (oldEventData.getSyncMode() != null) {
        newEventData.setSyncMode(oldEventData.getSyncMode());
    }
    if (oldEventData.isRemedy()) {
        newEventData.setRemedy(oldEventData.isRemedy());
    }
    newEventData.setSize(oldEventData.getSize() + newEventData.getSize());
    return newEventData;
}

Example 2 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class GroupExtractor method groupFilter.

private void groupFilter(EventData eventData, ColumnGroup columnGroup) {
    List<EventColumn> addColumns = new ArrayList<EventColumn>();
    // 判断一下是否存在字段组内字段的变更
    Set<String> updatedColumns = new HashSet<String>();
    Set<String> pks = new HashSet<String>();
    // 注意，这里只拿实际需要同步变更的字段
    for (EventColumn column : eventData.getUpdatedColumns()) {
        updatedColumns.add(column.getColumnName());
    }
    for (EventColumn pk : eventData.getKeys()) {
        pks.add(pk.getColumnName());
    }
    if (!CollectionUtils.isEmpty(eventData.getOldKeys())) {
        // 处理变更的主键
        int i = 0;
        for (EventColumn pk : eventData.getKeys()) {
            if (!StringUtils.equals(pk.getColumnValue(), eventData.getOldKeys().get(i).getColumnValue())) {
                updatedColumns.add(pk.getColumnName());
            }
            i++;
        }
    }
    if (containsInGroupColumn(updatedColumns, columnGroup.getColumnPairs())) {
        // 将变更的字段+变更的主键 去和 group字段进行交集处理
        for (ColumnPair columnPair : columnGroup.getColumnPairs()) {
            // 原谅我起这么长的变量名…
            boolean groupColumnHasInChangedColunms = false;
            // for (String columnName : updatedColumns) {
            for (EventColumn column : eventData.getColumns()) {
                if (StringUtils.equalsIgnoreCase(columnPair.getSourceColumn().getName(), column.getColumnName())) {
                    groupColumnHasInChangedColunms = true;
                    if (!column.isUpdate()) {
                        // 如果为非同步字段，强制修改为update=true进行数据同步
                        column.setUpdate(true);
                    }
                    break;
                }
            }
            if (!groupColumnHasInChangedColunms) {
                // 不存在对应的变更字段记录
                String columnName = columnPair.getSourceColumn().getName();
                if (!pks.contains(columnName)) {
                    // 只添加非主键的值到反查column，因为主键不需要反查
                    EventColumn addColumn = new EventColumn();
                    addColumn.setColumnName(columnPair.getSourceColumn().getName());
                    addColumn.setUpdate(true);
                    addColumns.add(addColumn);
                }
            }
        }
        if (!CollectionUtils.isEmpty(addColumns)) {
            // 字段去重
            // 添加不足的字段
            eventData.getColumns().addAll(addColumns);
            eventData.setSyncConsistency(SyncConsistency.MEDIA);
            return;
        }
    }
}

Example 3 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class ViewExtractor method columnFilter.

private List<EventColumn> columnFilter(List<EventColumn> eventColumns, List<ColumnPair> columnPairs, ColumnPairMode mode) {
    if (mode == null) {
        mode = ColumnPairMode.INCLUDE;
    }
    List<EventColumn> tempColumns = new ArrayList<EventColumn>();
    Map<String, ColumnPair> viewNames = new HashMap<String, ColumnPair>();
    for (ColumnPair columnPair : columnPairs) {
        viewNames.put(StringUtils.lowerCase(columnPair.getSourceColumn().getName()), columnPair);
    }
    for (EventColumn eventColumn : eventColumns) {
        if (mode.isInclude() && viewNames.containsKey(StringUtils.lowerCase(eventColumn.getColumnName()))) {
            // 正向匹配
            tempColumns.add(eventColumn);
        } else if (mode.isExclude() && !viewNames.containsKey(StringUtils.lowerCase(eventColumn.getColumnName()))) {
            // 逆向匹配
            tempColumns.add(eventColumn);
        }
    }
    return tempColumns;
}

Example 4 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class FileExtractor method getSingleRowFileInfos.

private List<FileData> getSingleRowFileInfos(long pairId, FileResolver fileResolver, EventData eventData) {
    if (eventData.getEventType() == EventType.DELETE && fileResolver.isDeleteRequired() == false) {
        return new ArrayList<FileData>();
    }
    Map<String, String> rowMap = new HashMap<String, String>();
    List<EventColumn> keyColumns = eventData.getKeys();
    List<EventColumn> eventColumns = eventData.getUpdatedColumns();
    for (EventColumn eventColumn : keyColumns) {
        rowMap.put(eventColumn.getColumnName().toUpperCase(), eventColumn.getColumnValue());
    }
    for (EventColumn eventColumn : eventColumns) {
        rowMap.put(eventColumn.getColumnName().toUpperCase(), eventColumn.getColumnValue());
    }
    FileInfo[] fileInfos = fileResolver.getFileInfo(rowMap);
    if (fileInfos == null || fileInfos.length == 0) {
        return new ArrayList<FileData>();
    } else {
        List<FileData> fileDatas = new ArrayList<FileData>();
        for (FileInfo fileInfo : fileInfos) {
            FileData fileData = new FileData();
            // 记录一下具体映射规则的id
            fileData.setPairId(pairId);
            fileData.setTableId(eventData.getTableId());
            fileData.setEventType(eventData.getEventType());
            fileData.setLastModifiedTime(fileInfo.getLastModifiedTime());
            fileData.setNameSpace(fileInfo.getNamespace());
            fileData.setPath(fileInfo.getPath());
            fileData.setSize(fileInfo.getSize());
            fileDatas.add(fileData);
        }
        return fileDatas;
    }
}

Example 5 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class SqlBuilderLoadInterceptor method buildColumnNames.

private String[] buildColumnNames(List<EventColumn> columns1, List<EventColumn> columns2) {
    String[] result = new String[columns1.size() + columns2.size()];
    int i = 0;
    for (i = 0; i < columns1.size(); i++) {
        EventColumn column = columns1.get(i);
        result[i] = column.getColumnName();
    }
    for (; i < columns1.size() + columns2.size(); i++) {
        EventColumn column = columns2.get(i - columns1.size());
        result[i] = column.getColumnName();
    }
    return result;
}