Examples with DbMapping com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping used on opensource projects

Example 1 with DbMapping

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping in project canal by alibaba.

the class RdbSyncService method getTargetColumnType.

/**
 * 获取目标字段类型
 *
 * @param conn sql connection
 * @param config 映射配置
 * @return 字段sqlType
 */
private Map<String, Integer> getTargetColumnType(Connection conn, MappingConfig config) {
    DbMapping dbMapping = config.getDbMapping();
    String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable();
    Map<String, Integer> columnType = columnsTypeCache.get(cacheKey);
    if (columnType == null) {
        synchronized (RdbSyncService.class) {
            columnType = columnsTypeCache.get(cacheKey);
            if (columnType == null) {
                columnType = new LinkedHashMap<>();
                final Map<String, Integer> columnTypeTmp = columnType;
                String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE 1=2";
                Util.sqlRS(conn, sql, rs -> {
                    try {
                        ResultSetMetaData rsd = rs.getMetaData();
                        int columnCount = rsd.getColumnCount();
                        for (int i = 1; i <= columnCount; i++) {
                            columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i));
                        }
                        columnsTypeCache.put(cacheKey, columnTypeTmp);
                    } catch (SQLException e) {
                        logger.error(e.getMessage(), e);
                    }
                });
            }
        }
    }
    return columnType;
}

Example 2 with DbMapping

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping in project canal by alibaba.

the class RdbEtlService method importData.

/**
 * 导入数据
 */
public EtlResult importData(List<String> params) {
    DbMapping dbMapping = config.getDbMapping();
    String sql = "SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable();
    return importData(sql, params);
}

Example 3 with DbMapping

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping in project canal by alibaba.

the class RdbSyncService method delete.

/**
 * 删除操作
 *
 * @param config
 * @param dml
 */
private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
    Map<String, Object> data = dml.getData();
    if (data == null || data.isEmpty()) {
        return;
    }
    DbMapping dbMapping = config.getDbMapping();
    Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
    StringBuilder sql = new StringBuilder();
    sql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE ");
    List<Map<String, ?>> values = new ArrayList<>();
    // 拼接主键
    appendCondition(dbMapping, sql, ctype, values, data);
    batchExecutor.execute(sql.toString(), values);
    if (logger.isTraceEnabled()) {
        logger.trace("Delete from target table, sql: {}", sql);
    }
}

Example 4 with DbMapping

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping in project canal by alibaba.

the class RdbSyncService method insert.

/**
 * 插入操作
 *
 * @param config 配置项
 * @param dml DML数据
 */
private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException {
    Map<String, Object> data = dml.getData();
    if (data == null || data.isEmpty()) {
        return;
    }
    DbMapping dbMapping = config.getDbMapping();
    Map<String, String> columnsMap = SyncUtil.getColumnsMap(dbMapping, data);
    StringBuilder insertSql = new StringBuilder();
    insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" (");
    columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append("`").append(targetColumnName).append("`").append(","));
    int len = insertSql.length();
    insertSql.delete(len - 1, len).append(") VALUES (");
    int mapLen = columnsMap.size();
    for (int i = 0; i < mapLen; i++) {
        insertSql.append("?,");
    }
    len = insertSql.length();
    insertSql.delete(len - 1, len).append(")");
    Map<String, Integer> ctype = getTargetColumnType(batchExecutor.getConn(), config);
    List<Map<String, ?>> values = new ArrayList<>();
    for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
        String targetColumnName = entry.getKey();
        String srcColumnName = entry.getValue();
        if (srcColumnName == null) {
            srcColumnName = Util.cleanColumn(targetColumnName);
        }
        Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase());
        if (type == null) {
            throw new RuntimeException("Target column: " + targetColumnName + " not matched");
        }
        Object value = data.get(srcColumnName);
        BatchExecutor.setValue(values, type, value);
    }
    try {
        batchExecutor.execute(insertSql.toString(), values);
    } catch (SQLException e) {
        if (skipDupException && (e.getMessage().contains("Duplicate entry") || e.getMessage().startsWith("ORA-00001:"))) {
        // ignore
        // TODO 增加更多关系数据库的主键冲突的错误码
        } else {
            throw e;
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Insert into target table, sql: {}", insertSql);
    }
}

Example 5 with DbMapping

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping in project canal by alibaba.

the class RdbSyncService method truncate.

/**
 * truncate操作
 *
 * @param config
 */
private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException {
    DbMapping dbMapping = config.getDbMapping();
    StringBuilder sql = new StringBuilder();
    sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping));
    batchExecutor.execute(sql.toString(), new ArrayList<>());
    if (logger.isTraceEnabled()) {
        logger.trace("Truncate target table, sql: {}", sql);
    }
}