Examples with Table org.apache.ddlutils.model.Table used on opensource projects

Example 11 with Table

use of org.apache.ddlutils.model.Table in project otter by alibaba.

the class FreedomExtractor method extract.

public void extract(DbBatch dbBatch) throws ExtractException {
    Assert.notNull(dbBatch);
    // 读取配置
    Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
    boolean skipFreedom = pipeline.getParameters().getSkipFreedom();
    String bufferSchema = pipeline.getParameters().getSystemSchema();
    String bufferTable = pipeline.getParameters().getSystemBufferTable();
    List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
    // 使用set，提升remove时的查找速度
    Set<EventData> removeDatas = new HashSet<EventData>();
    for (EventData eventData : eventDatas) {
        if (StringUtils.equalsIgnoreCase(bufferSchema, eventData.getSchemaName()) && StringUtils.equalsIgnoreCase(bufferTable, eventData.getTableName())) {
            if (eventData.getEventType().isDdl()) {
                continue;
            }
            if (skipFreedom) {
                // 判断是否需要忽略
                removeDatas.add(eventData);
                continue;
            }
            // 只处理insert / update记录
            if (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate()) {
                // 重新改写一下EventData的数据，根据系统表的定义
                EventColumn tableIdColumn = getMatchColumn(eventData.getColumns(), TABLE_ID);
                // 获取到对应tableId的media信息
                try {
                    DataMedia dataMedia = null;
                    Long tableId = Long.valueOf(tableIdColumn.getColumnValue());
                    eventData.setTableId(tableId);
                    if (tableId <= 0) {
                        // 直接按照full_name进行查找
                        // 尝试直接根据schema+table name进行查找
                        EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                        if (fullNameColumn != null) {
                            String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                            if (names.length >= 2) {
                                dataMedia = ConfigHelper.findSourceDataMedia(pipeline, names[0], names[1]);
                                eventData.setTableId(dataMedia.getId());
                            } else {
                                throw new ConfigException("no such DataMedia " + names);
                            }
                        }
                    } else {
                        // 如果指定了tableId，需要按照tableId进行严格查找，如果没找到，那说明不需要进行同步
                        dataMedia = ConfigHelper.findDataMedia(pipeline, Long.valueOf(tableIdColumn.getColumnValue()));
                    }
                    DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
                    // 考虑offer[1-128]的配置模式
                    if (!dataMedia.getNameMode().getMode().isSingle() || !dataMedia.getNamespaceMode().getMode().isSingle()) {
                        boolean hasError = true;
                        EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                        if (fullNameColumn != null) {
                            String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                            if (names.length >= 2) {
                                eventData.setSchemaName(names[0]);
                                eventData.setTableName(names[1]);
                                hasError = false;
                            }
                        }
                        if (hasError) {
                            // 出现异常，需要记录一下
                            logger.warn("dataMedia mode:{} , fullname:{} ", dataMedia.getMode(), fullNameColumn == null ? null : fullNameColumn.getColumnValue());
                            removeDatas.add(eventData);
                            // 跳过这条记录
                            continue;
                        }
                    } else {
                        eventData.setSchemaName(dataMedia.getNamespace());
                        eventData.setTableName(dataMedia.getName());
                    }
                    // 更新业务类型
                    EventColumn typeColumn = getMatchColumn(eventData.getColumns(), TYPE);
                    EventType eventType = EventType.valuesOf(typeColumn.getColumnValue());
                    eventData.setEventType(eventType);
                    if (eventType.isUpdate()) {
                        // 如果是update强制修改为insert，这样可以在目标端执行merge
                        // sql
                        eventData.setEventType(EventType.INSERT);
                    } else if (eventType.isDdl()) {
                        dbDialect.reloadTable(eventData.getSchemaName(), eventData.getTableName());
                        // 删除当前记录
                        removeDatas.add(eventData);
                        continue;
                    }
                    // 重新构建新的业务主键字段
                    EventColumn pkDataColumn = getMatchColumn(eventData.getColumns(), PK_DATA);
                    String pkData = pkDataColumn.getColumnValue();
                    String[] pks = StringUtils.split(pkData, PK_SPLIT);
                    Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                    List<EventColumn> newColumns = new ArrayList<EventColumn>();
                    Column[] primaryKeyColumns = table.getPrimaryKeyColumns();
                    if (primaryKeyColumns.length > pks.length) {
                        throw new ExtractException("data pk column size not match , data:" + eventData.toString());
                    }
                    // 构建字段
                    Column[] allColumns = table.getColumns();
                    int pkIndex = 0;
                    for (int i = 0; i < allColumns.length; i++) {
                        Column column = allColumns[i];
                        if (column.isPrimaryKey()) {
                            EventColumn newColumn = new EventColumn();
                            // 设置下标
                            newColumn.setIndex(i);
                            newColumn.setColumnName(column.getName());
                            newColumn.setColumnType(column.getTypeCode());
                            newColumn.setColumnValue(pks[pkIndex]);
                            newColumn.setKey(true);
                            newColumn.setNull(pks[pkIndex] == null);
                            newColumn.setUpdate(true);
                            // 添加到记录
                            newColumns.add(newColumn);
                            pkIndex++;
                        }
                    }
                    // 设置数据
                    eventData.setKeys(newColumns);
                    eventData.setOldKeys(new ArrayList<EventColumn>());
                    eventData.setColumns(new ArrayList<EventColumn>());
                    // 设置为行记录+反查
                    eventData.setSyncMode(SyncMode.ROW);
                    eventData.setSyncConsistency(SyncConsistency.MEDIA);
                    eventData.setRemedy(true);
                    // 默认为1kb，如果还是按照binlog大小计算的话，可能会采用rpc传输，导致内存不够用
                    eventData.setSize(1024);
                } catch (ConfigException e) {
                    // 忽略掉，因为系统表会被共享，所以这条记录会被不是该同步通道给获取到
                    logger.info("find DataMedia error " + eventData.toString(), e);
                    removeDatas.add(eventData);
                    continue;
                } catch (Throwable e) {
                    // 出现异常时忽略掉
                    logger.warn("process freedom data error " + eventData.toString(), e);
                    removeDatas.add(eventData);
                    continue;
                }
            } else {
                // 删除该记录
                removeDatas.add(eventData);
            }
        }
    }
    if (!CollectionUtils.isEmpty(removeDatas)) {
        eventDatas.removeAll(removeDatas);
    }
}

Example 12 with Table

use of org.apache.ddlutils.model.Table in project otter by alibaba.

the class MysqlDialect method initShardColumns.

private void initShardColumns() {
    // soft引用设置，避免内存爆了
    GenericMapMaker mapMaker = null;
    mapMaker = new MapMaker().softValues().evictionListener(new MapEvictionListener<List<String>, Table>() {

        public void onEviction(List<String> names, Table table) {
            logger.warn("Eviction For Table:" + table);
        }
    });
    this.shardColumns = mapMaker.makeComputingMap(new Function<List<String>, String>() {

        public String apply(List<String> names) {
            Assert.isTrue(names.size() == 2);
            try {
                String result = DdlUtils.getShardKeyByDRDS(jdbcTemplate, names.get(0), names.get(0), names.get(1));
                if (StringUtils.isEmpty(result)) {
                    return "";
                } else {
                    return result;
                }
            } catch (Exception e) {
                throw new NestableRuntimeException("find table [" + names.get(0) + "." + names.get(1) + "] error", e);
            }
        }
    });
}

Example 13 with Table

use of org.apache.ddlutils.model.Table in project otter by alibaba.

the class MessageParser method internParse.

/**
     * 解析出从canal中获取的Event事件<br>
     * Oracle:有变更的列值. <br>
     * <i>insert:从afterColumns中获取所有的变更数据<br>
     * <i>delete:从beforeColumns中获取所有的变更数据<br>
     * <i>update:在before中存放所有的主键和变化前的非主键值，在after中存放变化后的主键和非主键值,如果是复合主键，只会存放变化的主键<br>
     * Mysql:可以得到所有变更前和变更后的数据.<br>
     * <i>insert:从afterColumns中获取所有的变更数据<br>
     * <i>delete:从beforeColumns中获取所有的变更数据<br>
     * <i>update:在beforeColumns中存放变更前的所有数据,在afterColumns中存放变更后的所有数据<br>
     */
private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChange, RowData rowData) {
    EventData eventData = new EventData();
    eventData.setTableName(entry.getHeader().getTableName());
    eventData.setSchemaName(entry.getHeader().getSchemaName());
    eventData.setEventType(EventType.valueOf(rowChange.getEventType().name()));
    eventData.setExecuteTime(entry.getHeader().getExecuteTime());
    EventType eventType = eventData.getEventType();
    TableInfoHolder tableHolder = null;
    if (!StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), eventData.getSchemaName())) {
        boolean useTableTransform = pipeline.getParameters().getUseTableTransform();
        Table table = null;
        DataMediaPair dataMediaPair = ConfigHelper.findDataMediaPairBySourceName(pipeline, eventData.getSchemaName(), eventData.getTableName());
        DataMedia dataMedia = dataMediaPair.getSource();
        eventData.setTableId(dataMedia.getId());
        // 获取目标表
        DataMedia targetDataMedia = dataMediaPair.getTarget();
        if (useTableTransform || dataMedia.getSource().getType().isOracle()) {
            // oracle需要反查一次meta
            // 如果设置了需要进行table meta转化，则反查一下table信息
            // 比如oracle erosa解析时可能使用了非物理主键，需要直接使用，信任erosa的信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
            // 查询一下meta信息
            table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
            if (table == null) {
                logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(), eventData.getTableName());
            }
            // 获取一下目标库的拆分字段,设置源表为主键
            // 首先要求源和目标的库名表名是一致的
            DbDialect targetDbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) targetDataMedia.getSource());
            if (targetDbDialect.isDRDS()) {
                String schemaName = buildName(eventData.getSchemaName(), dataMedia.getNamespaceMode(), targetDataMedia.getNamespaceMode());
                String tableName = buildName(eventData.getSchemaName(), dataMedia.getNameMode(), targetDataMedia.getNameMode());
                String shardColumns = targetDbDialect.getShardColumns(schemaName, tableName);
                if (StringUtils.isNotEmpty(shardColumns)) {
                    String[] columns = StringUtils.split(shardColumns, ',');
                    for (String key : columns) {
                        org.apache.ddlutils.model.Column col = table.findColumn(key, false);
                        if (col != null) {
                            col.setPrimaryKey(true);
                        } else {
                            logger.warn(String.format("shardColumn %s in table[%s.%s] is not found", key, eventData.getSchemaName(), eventData.getTableName()));
                        }
                    }
                }
            }
            tableHolder = new TableInfoHolder(dbDialect, table, useTableTransform);
        }
    }
    List<Column> beforeColumns = rowData.getBeforeColumnsList();
    List<Column> afterColumns = rowData.getAfterColumnsList();
    String tableName = eventData.getSchemaName() + "." + eventData.getTableName();
    // 判断一下是否需要all columns
    // 如果是rowMode模式，所有字段都需要标记为updated
    boolean isRowMode = pipeline.getParameters().getSyncMode().isRow();
    boolean needAllColumns = isRowMode || checkNeedAllColumns(pipeline);
    // 变更后的主键
    Map<String, EventColumn> keyColumns = new LinkedHashMap<String, EventColumn>();
    // 变更前的主键
    Map<String, EventColumn> oldKeyColumns = new LinkedHashMap<String, EventColumn>();
    // 有变化的非主键
    Map<String, EventColumn> notKeyColumns = new LinkedHashMap<String, EventColumn>();
    if (eventType.isInsert()) {
        for (Column column : afterColumns) {
            if (isKey(tableHolder, tableName, column)) {
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                // mysql 有效
                notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            }
        }
    } else if (eventType.isDelete()) {
        for (Column column : beforeColumns) {
            if (isKey(tableHolder, tableName, column)) {
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                // mysql 有效
                notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            }
        }
    } else if (eventType.isUpdate()) {
        // 获取变更前的主键.
        for (Column column : beforeColumns) {
            if (isKey(tableHolder, tableName, column)) {
                oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                // 同时记录一下new
                // key,因为mysql5.6之后出现了minimal模式,after里会没有主键信息,需要在before记录中找
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
                    // 针对行记录同步时，针对oracle记录一下非主键的字段，因为update时针对未变更的字段在aftercolume里没有
                    notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode, tableHolder));
                }
            }
        }
        for (Column column : afterColumns) {
            if (isKey(tableHolder, tableName, column)) {
                // 获取变更后的主键
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else if (needAllColumns || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE || column.getUpdated()) {
                // 在update操作时，oracle和mysql存放变更的非主键值的方式不同,oracle只有变更的字段;
                // mysql会把变更前和变更后的字段都发出来，只需要取有变更的字段.
                // 如果是oracle库，after里一定为对应的变更字段
                boolean isUpdate = true;
                if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) {
                    // mysql的after里部分数据为未变更,oracle里after里为变更字段
                    isUpdate = column.getUpdated();
                }
                // 如果是rowMode，所有字段都为updated
                notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode || isUpdate, tableHolder));
            }
        }
        if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
            // 针对oracle进行特殊处理
            checkUpdateKeyColumns(oldKeyColumns, keyColumns);
        }
    }
    List<EventColumn> keys = new ArrayList<EventColumn>(keyColumns.values());
    List<EventColumn> oldKeys = new ArrayList<EventColumn>(oldKeyColumns.values());
    List<EventColumn> columns = new ArrayList<EventColumn>(notKeyColumns.values());
    Collections.sort(keys, new EventColumnIndexComparable());
    Collections.sort(oldKeys, new EventColumnIndexComparable());
    Collections.sort(columns, new EventColumnIndexComparable());
    if (!keyColumns.isEmpty()) {
        eventData.setKeys(keys);
        if (eventData.getEventType().isUpdate() && !oldKeys.equals(keys)) {
            // update类型，如果存在主键不同,则记录下old
            // keys为变更前的主键
            eventData.setOldKeys(oldKeys);
        }
        eventData.setColumns(columns);
    // } else if (CanalEntry.Type.MYSQL ==
    // entry.getHeader().getSourceType()) {
    // // 只支持mysql无主键同步
    // if (eventType.isUpdate()) {
    // List<EventColumn> oldColumns = new ArrayList<EventColumn>();
    // List<EventColumn> newColumns = new ArrayList<EventColumn>();
    // for (Column column : beforeColumns) {
    // oldColumns.add(copyEventColumn(column, true, tableHolder));
    // }
    //
    // for (Column column : afterColumns) {
    // newColumns.add(copyEventColumn(column, true, tableHolder));
    // }
    // Collections.sort(oldColumns, new EventColumnIndexComparable());
    // Collections.sort(newColumns, new EventColumnIndexComparable());
    // eventData.setOldKeys(oldColumns);// 做为老主键
    // eventData.setKeys(newColumns);// 做为新主键，需要保证新老主键字段数量一致
    // } else {
    // // 针对无主键，等同为所有都是主键进行处理
    // eventData.setKeys(columns);
    // }
    } else {
        throw new SelectException("this rowdata has no pks , entry: " + entry.toString() + " and rowData: " + rowData);
    }
    return eventData;
}

Example 14 with Table

use of org.apache.ddlutils.model.Table in project otter by alibaba.

the class DbDialectIntegration method test_mysql.

@Test(expectedExceptions = RuntimeException.class)
public void test_mysql() {
    DbMediaSource dbMediaSource = new DbMediaSource();
    dbMediaSource.setId(10L);
    dbMediaSource.setDriver("com.mysql.jdbc.Driver");
    dbMediaSource.setUsername("xxxxx");
    dbMediaSource.setPassword("xxxxx");
    dbMediaSource.setUrl("jdbc:mysql://127.0.0.1:3306");
    dbMediaSource.setEncode("UTF-8");
    dbMediaSource.setType(DataMediaType.MYSQL);
    final DbDialect dbDialect = dbDialectFactory.getDbDialect(2L, dbMediaSource);
    want.object(dbDialect).clazIs(MysqlDialect.class);
    Table table = dbDialect.findTable("test", "ljh_demo");
    System.out.println(table);
    final SqlTemplate sqlTemplate = dbDialect.getSqlTemplate();
    final JdbcTemplate jdbcTemplate = dbDialect.getJdbcTemplate();
    final TransactionTemplate transactionTemplate = dbDialect.getTransactionTemplate();
    final int[] pkColumnTypes = { Types.INTEGER };
    final int[] columnTypes = { Types.VARCHAR, Types.INTEGER, Types.DECIMAL, Types.BIGINT };
    transactionTemplate.execute(new TransactionCallback() {

        public Object doInTransaction(TransactionStatus status) {
            int affect = 0;
            String sql = null;
            // 执行insert
            sql = sqlTemplate.getInsertSql(SCHEMA_NAME, TABLE_NAME, pkColumns, columns);
            System.out.println(sql);
            affect = (Integer) jdbcTemplate.execute(sql, new PreparedStatementCallback() {

                public Object doInPreparedStatement(PreparedStatement ps) throws SQLException, DataAccessException {
                    doPreparedStatement(ps, dbDialect, toTypes(columnTypes, pkColumnTypes), toValues(columnValues, pkColumnValues));
                    return ps.executeUpdate();
                }
            });
            want.number(affect).isEqualTo(1);
            throw new RuntimeException("rollback");
        }
    });
}

Example 15 with Table

use of org.apache.ddlutils.model.Table in project otter by alibaba.

the class DbDialectTableTest method testMysqlTable.

@Test
public void testMysqlTable() {
    DbDataMedia mysqlMedia = getMysqlMedia();
    DbDialect dbDialect = dbDialectFactory.getDbDialect(1L, mysqlMedia.getSource());
    Table table = dbDialect.findTable(mysqlMedia.getNamespace(), mysqlMedia.getName());
    want.object(table).notNull();
    System.out.println("tableName = " + table.getName());
    Column[] columns = table.getColumns();
    for (Column column : columns) {
        System.out.println("columnName = " + column.getName() + ",columnType = " + column.getTypeCode() + ",isPrimary = " + column.isPrimaryKey() + ",nullable = " + column.isRequired());
    }
}