Examples with EventType com.alibaba.otter.shared.etl.model.EventType used on opensource projects

Example 1 with EventType

use of com.alibaba.otter.shared.etl.model.EventType in project otter by alibaba.

the class FileBatchConflictDetectServiceImpl method onFileConflictDetect.

/**
     * 具体冲突检测的行为
     */
private FileBatch onFileConflictDetect(FileConflictDetectEvent event) {
    final FileBatch fileBatch = event.getFileBatch();
    if (CollectionUtils.isEmpty(fileBatch.getFiles())) {
        return fileBatch;
    }
    ExecutorTemplate executorTemplate = executorTemplateGetter.get();
    try {
        MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
        executorTemplate.start();
        // 重新设置下poolSize
        Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId());
        executorTemplate.adjustPoolSize(pipeline.getParameters().getFileLoadPoolSize());
        // 启动
        final List<FileData> result = Collections.synchronizedList(new ArrayList<FileData>());
        final List<FileData> filter = Collections.synchronizedList(new ArrayList<FileData>());
        for (final FileData source : fileBatch.getFiles()) {
            EventType type = source.getEventType();
            if (type.isDelete()) {
                result.add(source);
            } else {
                executorTemplate.submit(new Runnable() {

                    public void run() {
                        MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
                        // 处理更新类型
                        String namespace = source.getNameSpace();
                        String path = source.getPath();
                        FileData target = null;
                        int count = 0;
                        while (count++ < retry) {
                            // 进行重试处理
                            try {
                                if (true == StringUtils.isBlank(namespace)) {
                                    // local file
                                    java.io.File targetFile = new java.io.File(path);
                                    if (true == targetFile.exists()) {
                                        // modified time cost
                                        long lastModified = targetFile.lastModified();
                                        long size = targetFile.length();
                                        // 更新数据
                                        target = new FileData();
                                        target.setLastModifiedTime(lastModified);
                                        target.setSize(size);
                                    }
                                } else {
                                    // remote file
                                    throw new RuntimeException(source + " is not support!");
                                }
                                // 不出异常就跳出
                                break;
                            } catch (Exception ex) {
                                target = null;
                            }
                        }
                        boolean shouldSync = false;
                        if (target != null) {
                            if (true == accept(target, source)) {
                                shouldSync = true;
                            }
                        } else {
                            shouldSync = true;
                        }
                        if (true == shouldSync) {
                            result.add(source);
                        } else {
                            filter.add(source);
                        }
                    }
                });
            }
        }
        // 等待所有都处理完成
        executorTemplate.waitForResult();
        if (pipeline.getParameters().getDumpEvent() && logger.isInfoEnabled()) {
            logger.info(FileloadDumper.dumpFilterFileDatas(fileBatch.getIdentity(), fileBatch.getFiles().size(), result.size(), filter));
        }
        // 构造返回结果
        FileBatch target = new FileBatch();
        target.setIdentity(fileBatch.getIdentity());
        target.setFiles(result);
        return target;
    } finally {
        if (executorTemplate != null) {
            executorTemplateGetter.release(executorTemplate);
        }
        MDC.remove(OtterConstants.splitPipelineLoadLogFileKey);
    }
}

Example 2 with EventType

use of com.alibaba.otter.shared.etl.model.EventType in project otter by alibaba.

the class SqlBuilderLoadInterceptor method before.

public boolean before(DbLoadContext context, EventData currentData) {
    // 初步构建sql
    DbDialect dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(), (DbMediaSource) context.getDataMediaSource());
    SqlTemplate sqlTemplate = dbDialect.getSqlTemplate();
    EventType type = currentData.getEventType();
    String sql = null;
    String schemaName = (currentData.isWithoutSchema() ? null : currentData.getSchemaName());
    // 注意insert/update语句对应的字段数序都是将主键排在后面
    if (type.isInsert()) {
        if (CollectionUtils.isEmpty(currentData.getColumns()) && sqlTemplate instanceof OracleSqlTemplate) {
            // 如果表为全主键，直接进行insert
            // sql
            sql = sqlTemplate.getInsertSql(schemaName, currentData.getTableName(), buildColumnNames(currentData.getKeys()), buildColumnNames(currentData.getColumns()));
        } else {
            sql = sqlTemplate.getMergeSql(schemaName, currentData.getTableName(), buildColumnNames(currentData.getKeys()), buildColumnNames(currentData.getColumns()), new String[] {}, !dbDialect.isDRDS());
        }
    } else if (type.isUpdate()) {
        // String[] keyColumns = buildColumnNames(currentData.getKeys());
        // String[] otherColumns =
        // buildColumnNames(currentData.getUpdatedColumns());
        // boolean existOldKeys = false;
        // for (String key : keyColumns) {
        // // 找一下otherColumns是否有主键，存在就代表有主键变更
        // if (ArrayUtils.contains(otherColumns, key)) {
        // existOldKeys = true;
        // break;
        // }
        // }
        boolean existOldKeys = !CollectionUtils.isEmpty(currentData.getOldKeys());
        boolean rowMode = context.getPipeline().getParameters().getSyncMode().isRow();
        String[] keyColumns = null;
        String[] otherColumns = null;
        if (existOldKeys) {
            // 需要考虑主键变更的场景
            // 构造sql如下：update table xxx set pk = newPK where pk = oldPk
            keyColumns = buildColumnNames(currentData.getOldKeys());
            otherColumns = buildColumnNames(currentData.getUpdatedColumns(), currentData.getKeys());
        } else {
            keyColumns = buildColumnNames(currentData.getKeys());
            otherColumns = buildColumnNames(currentData.getUpdatedColumns());
        }
        if (rowMode && !existOldKeys) {
            // 如果是行记录,并且不存在主键变更，考虑merge sql
            sql = sqlTemplate.getMergeSql(schemaName, currentData.getTableName(), keyColumns, otherColumns, new String[] {}, !dbDialect.isDRDS());
        } else {
            // 否则进行update sql
            sql = sqlTemplate.getUpdateSql(schemaName, currentData.getTableName(), keyColumns, otherColumns);
        }
    } else if (type.isDelete()) {
        sql = sqlTemplate.getDeleteSql(schemaName, currentData.getTableName(), buildColumnNames(currentData.getKeys()));
    }
    // 处理下hint sql
    if (currentData.getHint() != null) {
        currentData.setSql(currentData.getHint() + sql);
    } else {
        currentData.setSql(sql);
    }
    return false;
}

Example 3 with EventType

use of com.alibaba.otter.shared.etl.model.EventType in project otter by alibaba.

the class FreedomExtractor method extract.

public void extract(DbBatch dbBatch) throws ExtractException {
    Assert.notNull(dbBatch);
    // 读取配置
    Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
    boolean skipFreedom = pipeline.getParameters().getSkipFreedom();
    String bufferSchema = pipeline.getParameters().getSystemSchema();
    String bufferTable = pipeline.getParameters().getSystemBufferTable();
    List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
    // 使用set，提升remove时的查找速度
    Set<EventData> removeDatas = new HashSet<EventData>();
    for (EventData eventData : eventDatas) {
        if (StringUtils.equalsIgnoreCase(bufferSchema, eventData.getSchemaName()) && StringUtils.equalsIgnoreCase(bufferTable, eventData.getTableName())) {
            if (eventData.getEventType().isDdl()) {
                continue;
            }
            if (skipFreedom) {
                // 判断是否需要忽略
                removeDatas.add(eventData);
                continue;
            }
            // 只处理insert / update记录
            if (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate()) {
                // 重新改写一下EventData的数据，根据系统表的定义
                EventColumn tableIdColumn = getMatchColumn(eventData.getColumns(), TABLE_ID);
                // 获取到对应tableId的media信息
                try {
                    DataMedia dataMedia = null;
                    Long tableId = Long.valueOf(tableIdColumn.getColumnValue());
                    eventData.setTableId(tableId);
                    if (tableId <= 0) {
                        // 直接按照full_name进行查找
                        // 尝试直接根据schema+table name进行查找
                        EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                        if (fullNameColumn != null) {
                            String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                            if (names.length >= 2) {
                                dataMedia = ConfigHelper.findSourceDataMedia(pipeline, names[0], names[1]);
                                eventData.setTableId(dataMedia.getId());
                            } else {
                                throw new ConfigException("no such DataMedia " + names);
                            }
                        }
                    } else {
                        // 如果指定了tableId，需要按照tableId进行严格查找，如果没找到，那说明不需要进行同步
                        dataMedia = ConfigHelper.findDataMedia(pipeline, Long.valueOf(tableIdColumn.getColumnValue()));
                    }
                    DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
                    // 考虑offer[1-128]的配置模式
                    if (!dataMedia.getNameMode().getMode().isSingle() || !dataMedia.getNamespaceMode().getMode().isSingle()) {
                        boolean hasError = true;
                        EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                        if (fullNameColumn != null) {
                            String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                            if (names.length >= 2) {
                                eventData.setSchemaName(names[0]);
                                eventData.setTableName(names[1]);
                                hasError = false;
                            }
                        }
                        if (hasError) {
                            // 出现异常，需要记录一下
                            logger.warn("dataMedia mode:{} , fullname:{} ", dataMedia.getMode(), fullNameColumn == null ? null : fullNameColumn.getColumnValue());
                            removeDatas.add(eventData);
                            // 跳过这条记录
                            continue;
                        }
                    } else {
                        eventData.setSchemaName(dataMedia.getNamespace());
                        eventData.setTableName(dataMedia.getName());
                    }
                    // 更新业务类型
                    EventColumn typeColumn = getMatchColumn(eventData.getColumns(), TYPE);
                    EventType eventType = EventType.valuesOf(typeColumn.getColumnValue());
                    eventData.setEventType(eventType);
                    if (eventType.isUpdate()) {
                        // 如果是update强制修改为insert，这样可以在目标端执行merge
                        // sql
                        eventData.setEventType(EventType.INSERT);
                    } else if (eventType.isDdl()) {
                        dbDialect.reloadTable(eventData.getSchemaName(), eventData.getTableName());
                        // 删除当前记录
                        removeDatas.add(eventData);
                        continue;
                    }
                    // 重新构建新的业务主键字段
                    EventColumn pkDataColumn = getMatchColumn(eventData.getColumns(), PK_DATA);
                    String pkData = pkDataColumn.getColumnValue();
                    String[] pks = StringUtils.split(pkData, PK_SPLIT);
                    Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                    List<EventColumn> newColumns = new ArrayList<EventColumn>();
                    Column[] primaryKeyColumns = table.getPrimaryKeyColumns();
                    if (primaryKeyColumns.length > pks.length) {
                        throw new ExtractException("data pk column size not match , data:" + eventData.toString());
                    }
                    // 构建字段
                    Column[] allColumns = table.getColumns();
                    int pkIndex = 0;
                    for (int i = 0; i < allColumns.length; i++) {
                        Column column = allColumns[i];
                        if (column.isPrimaryKey()) {
                            EventColumn newColumn = new EventColumn();
                            // 设置下标
                            newColumn.setIndex(i);
                            newColumn.setColumnName(column.getName());
                            newColumn.setColumnType(column.getTypeCode());
                            newColumn.setColumnValue(pks[pkIndex]);
                            newColumn.setKey(true);
                            newColumn.setNull(pks[pkIndex] == null);
                            newColumn.setUpdate(true);
                            // 添加到记录
                            newColumns.add(newColumn);
                            pkIndex++;
                        }
                    }
                    // 设置数据
                    eventData.setKeys(newColumns);
                    eventData.setOldKeys(new ArrayList<EventColumn>());
                    eventData.setColumns(new ArrayList<EventColumn>());
                    // 设置为行记录+反查
                    eventData.setSyncMode(SyncMode.ROW);
                    eventData.setSyncConsistency(SyncConsistency.MEDIA);
                    eventData.setRemedy(true);
                    // 默认为1kb，如果还是按照binlog大小计算的话，可能会采用rpc传输，导致内存不够用
                    eventData.setSize(1024);
                } catch (ConfigException e) {
                    // 忽略掉，因为系统表会被共享，所以这条记录会被不是该同步通道给获取到
                    logger.info("find DataMedia error " + eventData.toString(), e);
                    removeDatas.add(eventData);
                    continue;
                } catch (Throwable e) {
                    // 出现异常时忽略掉
                    logger.warn("process freedom data error " + eventData.toString(), e);
                    removeDatas.add(eventData);
                    continue;
                }
            } else {
                // 删除该记录
                removeDatas.add(eventData);
            }
        }
    }
    if (!CollectionUtils.isEmpty(removeDatas)) {
        eventDatas.removeAll(removeDatas);
    }
}

Example 4 with EventType

use of com.alibaba.otter.shared.etl.model.EventType in project otter by alibaba.

the class DbLoadAction method processStat.

private void processStat(EventData data, DbLoadContext context) {
    LoadThroughput throughput = loadStatsTracker.getStat(context.getIdentity());
    LoadCounter counter = throughput.getStat(data.getPairId());
    EventType type = data.getEventType();
    if (type.isInsert()) {
        counter.getInsertCount().incrementAndGet();
    } else if (type.isUpdate()) {
        counter.getUpdateCount().incrementAndGet();
    } else if (type.isDelete()) {
        counter.getDeleteCount().incrementAndGet();
    }
    counter.getRowCount().incrementAndGet();
    counter.getRowSize().addAndGet(calculateSize(data));
}

Example 5 with EventType

use of com.alibaba.otter.shared.etl.model.EventType in project otter by alibaba.

the class DbLoadData method merge.

public void merge(EventData data) {
    TableLoadData tableData = findTableData(data.getTableId());
    EventType type = data.getEventType();
    if (type.isInsert()) {
        tableData.getInsertDatas().add(data);
    } else if (type.isUpdate()) {
        tableData.getUpadateDatas().add(data);
    } else if (type.isDelete()) {
        tableData.getDeleteDatas().add(data);
    }
}