Examples with MappingConfig com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig used on opensource projects

Example 1 with MappingConfig

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig in project canal by alibaba.

the class RdbAdapter method etl.

/**
 * ETL方法
 *
 * @param task 任务名, 对应配置名
 * @param params etl筛选条件
 * @return ETL结果
 */
@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    MappingConfig config = rdbMapping.get(task);
    RdbEtlService rdbEtlService = new RdbEtlService(dataSource, config);
    if (config != null) {
        return rdbEtlService.importData(params);
    } else {
        StringBuilder resultMsg = new StringBuilder();
        boolean resSucc = true;
        for (MappingConfig configTmp : rdbMapping.values()) {
            // 取所有的destination为task的配置
            if (configTmp.getDestination().equals(task)) {
                EtlResult etlRes = rdbEtlService.importData(params);
                if (!etlRes.getSucceeded()) {
                    resSucc = false;
                    resultMsg.append(etlRes.getErrorMessage()).append("\n");
                } else {
                    resultMsg.append(etlRes.getResultMessage()).append("\n");
                }
            }
        }
        if (resultMsg.length() > 0) {
            etlResult.setSucceeded(resSucc);
            if (resSucc) {
                etlResult.setResultMessage(resultMsg.toString());
            } else {
                etlResult.setErrorMessage(resultMsg.toString());
            }
            return etlResult;
        }
    }
    etlResult.setSucceeded(false);
    etlResult.setErrorMessage("Task not found");
    return etlResult;
}

Example 2 with MappingConfig

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig in project canal by alibaba.

the class RdbAdapter method init.

/**
 * 初始化方法
 *
 * @param configuration 外部适配器配置信息
 */
@Override
public void init(OuterAdapterConfig configuration, Properties envProperties) {
    this.envProperties = envProperties;
    Map<String, MappingConfig> rdbMappingTmp = ConfigLoader.load(envProperties);
    // 过滤不匹配的key的配置
    rdbMappingTmp.forEach((key, mappingConfig) -> {
        if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
            rdbMapping.put(key, mappingConfig);
        }
    });
    if (rdbMapping.isEmpty()) {
        throw new RuntimeException("No rdb adapter found for config key: " + configuration.getKey());
    }
    for (Map.Entry<String, MappingConfig> entry : rdbMapping.entrySet()) {
        String configName = entry.getKey();
        MappingConfig mappingConfig = entry.getValue();
        if (!mappingConfig.getDbMapping().getMirrorDb()) {
            String key;
            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable();
            } else {
                key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_" + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable();
            }
            Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(key, k1 -> new ConcurrentHashMap<>());
            configMap.put(configName, mappingConfig);
        } else {
            // mirrorDB
            String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping().getDatabase();
            mirrorDbConfigCache.put(key, MirrorDbConfig.create(configName, mappingConfig));
        }
    }
    // 初始化连接池
    Map<String, String> properties = configuration.getProperties();
    dataSource = new DruidDataSource();
    dataSource.setDriverClassName(properties.get("jdbc.driverClassName"));
    dataSource.setUrl(properties.get("jdbc.url"));
    dataSource.setUsername(properties.get("jdbc.username"));
    dataSource.setPassword(properties.get("jdbc.password"));
    dataSource.setInitialSize(1);
    dataSource.setMinIdle(1);
    dataSource.setMaxActive(30);
    dataSource.setMaxWait(60000);
    dataSource.setTimeBetweenEvictionRunsMillis(60000);
    dataSource.setMinEvictableIdleTimeMillis(300000);
    dataSource.setUseUnfairLock(true);
    try {
        dataSource.init();
    } catch (SQLException e) {
        logger.error("ERROR ## failed to initial datasource: " + properties.get("jdbc.url"), e);
    }
    String threads = properties.get("threads");
    // String commitSize = properties.get("commitSize");
    boolean skipDupException = BooleanUtils.toBoolean(configuration.getProperties().getOrDefault("skipDupException", "true"));
    rdbSyncService = new RdbSyncService(dataSource, threads != null ? Integer.valueOf(threads) : null, skipDupException);
    rdbMirrorDbSyncService = new RdbMirrorDbSyncService(mirrorDbConfigCache, dataSource, threads != null ? Integer.valueOf(threads) : null, rdbSyncService.getColumnsTypeCache(), skipDupException);
    rdbConfigMonitor = new RdbConfigMonitor();
    rdbConfigMonitor.init(configuration.getKey(), this, envProperties);
}

Example 3 with MappingConfig

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig in project canal by alibaba.

the class RdbSyncService method sync.

/**
 * 批量同步
 *
 * @param mappingConfig 配置集合
 * @param dmls 批量 DML
 */
public void sync(Map<String, Map<String, MappingConfig>> mappingConfig, List<Dml> dmls, Properties envProperties) {
    sync(dmls, dml -> {
        if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
            // DDL
            columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
            return false;
        } else {
            // DML
            String destination = StringUtils.trimToEmpty(dml.getDestination());
            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
            String database = dml.getDatabase();
            String table = dml.getTable();
            Map<String, MappingConfig> configMap;
            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
                configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table);
            } else {
                configMap = mappingConfig.get(destination + "_" + database + "-" + table);
            }
            if (configMap == null) {
                return false;
            }
            if (configMap.values().isEmpty()) {
                return false;
            }
            for (MappingConfig config : configMap.values()) {
                boolean caseInsensitive = config.getDbMapping().isCaseInsensitive();
                if (config.getConcurrent()) {
                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
                    singleDmls.forEach(singleDml -> {
                        int hash = pkHash(config.getDbMapping(), singleDml.getData());
                        SyncItem syncItem = new SyncItem(config, singleDml);
                        dmlsPartition[hash].add(syncItem);
                    });
                } else {
                    int hash = 0;
                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml, caseInsensitive);
                    singleDmls.forEach(singleDml -> {
                        SyncItem syncItem = new SyncItem(config, singleDml);
                        dmlsPartition[hash].add(syncItem);
                    });
                }
            }
            return true;
        }
    });
}

Example 4 with MappingConfig

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig in project canal by alibaba.

the class RdbAdapter method count.

/**
 * 获取总数方法
 *
 * @param task 任务名, 对应配置名
 * @return 总数
 */
@Override
public Map<String, Object> count(String task) {
    MappingConfig config = rdbMapping.get(task);
    MappingConfig.DbMapping dbMapping = config.getDbMapping();
    String sql = "SELECT COUNT(1) AS cnt FROM " + SyncUtil.getDbTableName(dbMapping);
    Connection conn = null;
    Map<String, Object> res = new LinkedHashMap<>();
    try {
        conn = dataSource.getConnection();
        Util.sqlRS(conn, sql, rs -> {
            try {
                if (rs.next()) {
                    Long rowCount = rs.getLong("cnt");
                    res.put("count", rowCount);
                }
            } catch (SQLException e) {
                logger.error(e.getMessage(), e);
            }
        });
    } catch (SQLException e) {
        logger.error(e.getMessage(), e);
    } finally {
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }
    res.put("targetTable", SyncUtil.getDbTableName(dbMapping));
    return res;
}

Example 5 with MappingConfig

use of com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig in project canal by alibaba.

the class RdbMirrorDbSyncService method initMappingConfig.

/**
 * 初始化表配置
 *
 * @param key 配置key: destination.database.table
 * @param baseConfigMap db sync config
 * @param dml DML
 */
private void initMappingConfig(String key, MappingConfig baseConfigMap, MirrorDbConfig mirrorDbConfig, Dml dml) {
    MappingConfig mappingConfig = mirrorDbConfig.getTableConfig().get(key);
    if (mappingConfig == null) {
        // 构造表配置
        mappingConfig = new MappingConfig();
        mappingConfig.setDataSourceKey(baseConfigMap.getDataSourceKey());
        mappingConfig.setDestination(baseConfigMap.getDestination());
        mappingConfig.setGroupId(baseConfigMap.getGroupId());
        mappingConfig.setOuterAdapterKey(baseConfigMap.getOuterAdapterKey());
        mappingConfig.setConcurrent(baseConfigMap.getConcurrent());
        MappingConfig.DbMapping dbMapping = new MappingConfig.DbMapping();
        mappingConfig.setDbMapping(dbMapping);
        dbMapping.setDatabase(dml.getDatabase());
        dbMapping.setTable(dml.getTable());
        dbMapping.setTargetDb(dml.getDatabase());
        dbMapping.setTargetTable(dml.getTable());
        dbMapping.setMapAll(true);
        List<String> pkNames = dml.getPkNames();
        Map<String, String> pkMapping = new LinkedHashMap<>();
        pkNames.forEach(pkName -> pkMapping.put(pkName, pkName));
        dbMapping.setTargetPk(pkMapping);
        mirrorDbConfig.getTableConfig().put(key, mappingConfig);
    }
}