Examples with Column org.apache.ddlutils.model.Column used on opensource projects

Example 6 with Column

use of org.apache.ddlutils.model.Column in project otter by alibaba.

the class FreedomExtractor method extract.

public void extract(DbBatch dbBatch) throws ExtractException {
    Assert.notNull(dbBatch);
    // 读取配置
    Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
    boolean skipFreedom = pipeline.getParameters().getSkipFreedom();
    String bufferSchema = pipeline.getParameters().getSystemSchema();
    String bufferTable = pipeline.getParameters().getSystemBufferTable();
    List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
    // 使用set，提升remove时的查找速度
    Set<EventData> removeDatas = new HashSet<EventData>();
    for (EventData eventData : eventDatas) {
        if (StringUtils.equalsIgnoreCase(bufferSchema, eventData.getSchemaName()) && StringUtils.equalsIgnoreCase(bufferTable, eventData.getTableName())) {
            if (eventData.getEventType().isDdl()) {
                continue;
            }
            if (skipFreedom) {
                // 判断是否需要忽略
                removeDatas.add(eventData);
                continue;
            }
            // 只处理insert / update记录
            if (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate()) {
                // 重新改写一下EventData的数据，根据系统表的定义
                EventColumn tableIdColumn = getMatchColumn(eventData.getColumns(), TABLE_ID);
                // 获取到对应tableId的media信息
                try {
                    DataMedia dataMedia = null;
                    Long tableId = Long.valueOf(tableIdColumn.getColumnValue());
                    eventData.setTableId(tableId);
                    if (tableId <= 0) {
                        // 直接按照full_name进行查找
                        // 尝试直接根据schema+table name进行查找
                        EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                        if (fullNameColumn != null) {
                            String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                            if (names.length >= 2) {
                                dataMedia = ConfigHelper.findSourceDataMedia(pipeline, names[0], names[1]);
                                eventData.setTableId(dataMedia.getId());
                            } else {
                                throw new ConfigException("no such DataMedia " + names);
                            }
                        }
                    } else {
                        // 如果指定了tableId，需要按照tableId进行严格查找，如果没找到，那说明不需要进行同步
                        dataMedia = ConfigHelper.findDataMedia(pipeline, Long.valueOf(tableIdColumn.getColumnValue()));
                    }
                    DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
                    // 考虑offer[1-128]的配置模式
                    if (!dataMedia.getNameMode().getMode().isSingle() || !dataMedia.getNamespaceMode().getMode().isSingle()) {
                        boolean hasError = true;
                        EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                        if (fullNameColumn != null) {
                            String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                            if (names.length >= 2) {
                                eventData.setSchemaName(names[0]);
                                eventData.setTableName(names[1]);
                                hasError = false;
                            }
                        }
                        if (hasError) {
                            // 出现异常，需要记录一下
                            logger.warn("dataMedia mode:{} , fullname:{} ", dataMedia.getMode(), fullNameColumn == null ? null : fullNameColumn.getColumnValue());
                            removeDatas.add(eventData);
                            // 跳过这条记录
                            continue;
                        }
                    } else {
                        eventData.setSchemaName(dataMedia.getNamespace());
                        eventData.setTableName(dataMedia.getName());
                    }
                    // 更新业务类型
                    EventColumn typeColumn = getMatchColumn(eventData.getColumns(), TYPE);
                    EventType eventType = EventType.valuesOf(typeColumn.getColumnValue());
                    eventData.setEventType(eventType);
                    if (eventType.isUpdate()) {
                        // 如果是update强制修改为insert，这样可以在目标端执行merge
                        // sql
                        eventData.setEventType(EventType.INSERT);
                    } else if (eventType.isDdl()) {
                        dbDialect.reloadTable(eventData.getSchemaName(), eventData.getTableName());
                        // 删除当前记录
                        removeDatas.add(eventData);
                        continue;
                    }
                    // 重新构建新的业务主键字段
                    EventColumn pkDataColumn = getMatchColumn(eventData.getColumns(), PK_DATA);
                    String pkData = pkDataColumn.getColumnValue();
                    String[] pks = StringUtils.split(pkData, PK_SPLIT);
                    Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                    List<EventColumn> newColumns = new ArrayList<EventColumn>();
                    Column[] primaryKeyColumns = table.getPrimaryKeyColumns();
                    if (primaryKeyColumns.length > pks.length) {
                        throw new ExtractException("data pk column size not match , data:" + eventData.toString());
                    }
                    // 构建字段
                    Column[] allColumns = table.getColumns();
                    int pkIndex = 0;
                    for (int i = 0; i < allColumns.length; i++) {
                        Column column = allColumns[i];
                        if (column.isPrimaryKey()) {
                            EventColumn newColumn = new EventColumn();
                            // 设置下标
                            newColumn.setIndex(i);
                            newColumn.setColumnName(column.getName());
                            newColumn.setColumnType(column.getTypeCode());
                            newColumn.setColumnValue(pks[pkIndex]);
                            newColumn.setKey(true);
                            newColumn.setNull(pks[pkIndex] == null);
                            newColumn.setUpdate(true);
                            // 添加到记录
                            newColumns.add(newColumn);
                            pkIndex++;
                        }
                    }
                    // 设置数据
                    eventData.setKeys(newColumns);
                    eventData.setOldKeys(new ArrayList<EventColumn>());
                    eventData.setColumns(new ArrayList<EventColumn>());
                    // 设置为行记录+反查
                    eventData.setSyncMode(SyncMode.ROW);
                    eventData.setSyncConsistency(SyncConsistency.MEDIA);
                    eventData.setRemedy(true);
                    // 默认为1kb，如果还是按照binlog大小计算的话，可能会采用rpc传输，导致内存不够用
                    eventData.setSize(1024);
                } catch (ConfigException e) {
                    // 忽略掉，因为系统表会被共享，所以这条记录会被不是该同步通道给获取到
                    logger.info("find DataMedia error " + eventData.toString(), e);
                    removeDatas.add(eventData);
                    continue;
                } catch (Throwable e) {
                    // 出现异常时忽略掉
                    logger.warn("process freedom data error " + eventData.toString(), e);
                    removeDatas.add(eventData);
                    continue;
                }
            } else {
                // 删除该记录
                removeDatas.add(eventData);
            }
        }
    }
    if (!CollectionUtils.isEmpty(removeDatas)) {
        eventDatas.removeAll(removeDatas);
    }
}

Example 7 with Column

use of org.apache.ddlutils.model.Column in project otter by alibaba.

the class DbDialectTableTest method testMysqlTable.

@Test
public void testMysqlTable() {
    DbDataMedia mysqlMedia = getMysqlMedia();
    DbDialect dbDialect = dbDialectFactory.getDbDialect(1L, mysqlMedia.getSource());
    Table table = dbDialect.findTable(mysqlMedia.getNamespace(), mysqlMedia.getName());
    want.object(table).notNull();
    System.out.println("tableName = " + table.getName());
    Column[] columns = table.getColumns();
    for (Column column : columns) {
        System.out.println("columnName = " + column.getName() + ",columnType = " + column.getTypeCode() + ",isPrimary = " + column.isPrimaryKey() + ",nullable = " + column.isRequired());
    }
}

Example 8 with Column

use of org.apache.ddlutils.model.Column in project otter by alibaba.

the class RowDataTransformer method translateColumn.

private EventColumn translateColumn(EventData data, EventColumn scolumn, TableInfoHolder tableHolder, DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames) {
    EventType type = data.getEventType();
    EventColumn tcolumn = new EventColumn();
    tcolumn.setNull(scolumn.getColumnValue() == null);
    // 左右两边的主键值必须保持一样，可以不为物理主键
    tcolumn.setKey(scolumn.isKey());
    tcolumn.setIndex(scolumn.getIndex());
    tcolumn.setUpdate(scolumn.isUpdate());
    String columnName = translateColumnName(scolumn.getColumnName(), dataMediaPair, translateColumnNames);
    if (StringUtils.isBlank(columnName)) {
        throw new TransformException("can't translate column name:" + scolumn.getColumnName() + "in pair:" + dataMediaPair.toString());
    }
    // 特殊处理
    // columnName = StringUtils.remove(columnName, "`"); //
    // 处理下特殊字符，eromanga给了错误的字段名
    tcolumn.setColumnName(columnName);
    // 不反查，直接使用源库的类型
    tcolumn.setColumnType(scolumn.getColumnType());
    if (tableHolder != null) {
        // modify by ljh at 2013-01-23
        // 双向同步新增字段，在一边加了字段后，虽然新的字段没有产生业务变化，但会因为某些原因导致传递了新的字段到T模块
        // 此时在目标库并不存在这个字段，导致一直挂起。ps. mysql新增字段时间不是一般的长
        // 所以，做了一个容错处理，针对目标库不存在的字段，如果变更记录在源库不存在变更，并且是null值的，允许丢弃该字段(其实最好还是要判断源库的column的defaultValue和当前值是否一致)
        boolean canColumnsNotExist = tableHolder.isEnableCompatibleMissColumn();
        if (type == EventType.UPDATE) {
            // 非变更字段，且当前值为null
            canColumnsNotExist &= !scolumn.isUpdate() && scolumn.isNull();
        } else if (type == EventType.INSERT) {
            // 当前值为null
            canColumnsNotExist &= scolumn.isNull();
        } else if (type == EventType.DELETE) {
            // 主键不允许不存在
            canColumnsNotExist &= !scolumn.isKey();
        }
        Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
        // 匹配字段为空，可能源库发生过DDL操作，目标库重新载入一下meta信息
        if (matchDbColumn == null) {
            // 尝试reload一下table meta
            // 获取目标库的表信息
            DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource();
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource);
            String schemaName = tableHolder.getTable().getSchema();
            if (StringUtils.isEmpty(schemaName)) {
                schemaName = tableHolder.getTable().getCatalog();
            }
            // 强制反查一次，并放入cache
            Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false);
            tableHolder.setTable(table);
            matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
            if (matchDbColumn == null) {
                if (canColumnsNotExist) {
                    return null;
                } else {
                    throw new TransformException(scolumn.getColumnName() + " is not found in " + table.toString() + " and source : " + dataMediaPair.getTarget().getNamespace() + "." + dataMediaPair.getTarget().getName());
                }
            }
        }
        if (tableHolder.isUseTableTransform()) {
            int sqlType = matchDbColumn.getTypeCode();
            tcolumn.setColumnType(sqlType);
        }
    }
    // if (dataMediaPair.getTarget().getSource().getType().isOracle()) {
    // // 特殊处理下oracle编码
    // String encodeValue = SqlUtils.encoding(scolumn.getColumnValue(),
    // scolumn.getColumnType(),
    // dataMediaPair.getSource().getSource().getEncode(),
    // dataMediaPair.getTarget().getSource().getEncode());
    // tcolumn.setColumnValue(encodeValue);
    // } else {
    // mysql编码转化已经在驱动层面上完成
    tcolumn.setColumnValue(scolumn.getColumnValue());
    // }
    // 删除映射关系，避免下次重复转换
    translateColumnNames.remove(scolumn.getColumnName(), columnName);
    return tcolumn;
}

Example 9 with Column

use of org.apache.ddlutils.model.Column in project otter by alibaba.

the class AbstractDbDialectTest method testFindTable.

@Test
public void testFindTable() throws Exception {
    DataSource dataSource = createDataSource("jdbc:oracle:thin:@127.0.0.1:1521:OINTEST", "otter1", "jonathan", "oracle.jdbc.OracleDriver", DataMediaType.ORACLE, "utf-8");
    JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
    Table table = DdlUtils.findTable(jdbcTemplate, "otter1".toUpperCase(), "otter1".toUpperCase(), "wytable3".toUpperCase());
    System.out.println("the tablename = " + table.getSchema() + "." + table.getName());
    Column[] columns = table.getColumns();
    for (Column column : columns) {
        System.out.println("columnName = " + column.getName() + ",columnType = " + column.getTypeCode() + ",isPrimary = " + column.isPrimaryKey() + ",nullable = " + column.isRequired());
    }
}

Example 10 with Column

use of org.apache.ddlutils.model.Column in project otter by alibaba.

the class DdlUtils method readColumn.

private static Column readColumn(DatabaseMetaDataWrapper metaData, Map<String, Object> values) throws SQLException {
    Column column = new Column();
    column.setName((String) values.get("COLUMN_NAME"));
    column.setDefaultValue((String) values.get("COLUMN_DEF"));
    column.setTypeCode(((Integer) values.get("DATA_TYPE")).intValue());
    String typeName = (String) values.get("TYPE_NAME");
    if ((typeName != null) && typeName.startsWith("TIMESTAMP")) {
        column.setTypeCode(Types.TIMESTAMP);
    }
    // modify 2013-09-25，处理下unsigned
    if ((typeName != null) && StringUtils.containsIgnoreCase(typeName, "UNSIGNED")) {
        // 如果为unsigned，往上调大一个量级，避免数据溢出
        switch(column.getTypeCode()) {
            case Types.TINYINT:
                column.setTypeCode(Types.SMALLINT);
                break;
            case Types.SMALLINT:
                column.setTypeCode(Types.INTEGER);
                break;
            case Types.INTEGER:
                column.setTypeCode(Types.BIGINT);
                break;
            case Types.BIGINT:
                column.setTypeCode(Types.DECIMAL);
                break;
            default:
                break;
        }
    }
    Integer precision = (Integer) values.get("NUM_PREC_RADIX");
    if (precision != null) {
        column.setPrecisionRadix(precision.intValue());
    }
    String size = (String) values.get("COLUMN_SIZE");
    if (size == null) {
        size = (String) _defaultSizes.get(new Integer(column.getTypeCode()));
    }
    // we're setting the size after the precision and radix in case
    // the database prefers to return them in the size value
    column.setSize(size);
    int scale = 0;
    Object dec_digits = values.get("DECIMAL_DIGITS");
    if (dec_digits instanceof String) {
        scale = (dec_digits == null) ? 0 : NumberUtils.toInt(dec_digits.toString());
    } else if (dec_digits instanceof Integer) {
        scale = (dec_digits == null) ? 0 : (Integer) dec_digits;
    }
    if (scale != 0) {
        column.setScale(scale);
    }
    column.setRequired("NO".equalsIgnoreCase(((String) values.get("IS_NULLABLE")).trim()));
    column.setDescription((String) values.get("REMARKS"));
    return column;
}