Examples with Configuration com.alibaba.datax.common.util.Configuration used on opensource projects

Example 31 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class ReaderSplitUtil method doPreCheckSplit.

public static Configuration doPreCheckSplit(Configuration originalSliceConfig) {
    Configuration queryConfig = originalSliceConfig.clone();
    boolean isTableMode = originalSliceConfig.getBool(Constant.IS_TABLE_MODE).booleanValue();
    String splitPK = originalSliceConfig.getString(Key.SPLIT_PK);
    String column = originalSliceConfig.getString(Key.COLUMN);
    String where = originalSliceConfig.getString(Key.WHERE, null);
    List<Object> conns = queryConfig.getList(Constant.CONN_MARK, Object.class);
    for (int i = 0, len = conns.size(); i < len; i++) {
        Configuration connConf = Configuration.from(conns.get(i).toString());
        List<String> querys = new ArrayList<String>();
        List<String> splitPkQuerys = new ArrayList<String>();
        String connPath = String.format("connection[%d]", i);
        // 说明是配置的 table 方式
        if (isTableMode) {
            // 已在之前进行了扩展和`处理，可以直接使用
            List<String> tables = connConf.getList(Key.TABLE, String.class);
            Validate.isTrue(null != tables && !tables.isEmpty(), "您读取数据库表配置错误.");
            for (String table : tables) {
                querys.add(SingleTableSplitUtil.buildQuerySql(column, table, where));
                if (splitPK != null && !splitPK.isEmpty()) {
                    splitPkQuerys.add(SingleTableSplitUtil.genPKSql(splitPK.trim(), table, where));
                }
            }
            if (!splitPkQuerys.isEmpty()) {
                connConf.set(Key.SPLIT_PK_SQL, splitPkQuerys);
            }
            connConf.set(Key.QUERY_SQL, querys);
            queryConfig.set(connPath, connConf);
        } else {
            // 说明是配置的 querySql 方式
            List<String> sqls = connConf.getList(Key.QUERY_SQL, String.class);
            for (String querySql : sqls) {
                querys.add(querySql);
            }
            connConf.set(Key.QUERY_SQL, querys);
            queryConfig.set(connPath, connConf);
        }
    }
    return queryConfig;
}

Example 32 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class OriginalConfPretreatmentUtil method simplifyConf.

public static void simplifyConf(Configuration originalConfig) {
    List<Object> connections = originalConfig.getList(Constant.CONN_MARK, Object.class);
    int tableNum = 0;
    for (int i = 0, len = connections.size(); i < len; i++) {
        Configuration connConf = Configuration.from(connections.get(i).toString());
        String jdbcUrl = connConf.getString(Key.JDBC_URL);
        if (StringUtils.isBlank(jdbcUrl)) {
            throw DataXException.asDataXException(DBUtilErrorCode.REQUIRED_VALUE, "您未配置的写入数据库表的 jdbcUrl.");
        }
        jdbcUrl = DATABASE_TYPE.appendJDBCSuffixForReader(jdbcUrl);
        originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.JDBC_URL), jdbcUrl);
        List<String> tables = connConf.getList(Key.TABLE, String.class);
        if (null == tables || tables.isEmpty()) {
            throw DataXException.asDataXException(DBUtilErrorCode.REQUIRED_VALUE, "您未配置写入数据库表的表名称. 根据配置DataX找不到您配置的表. 请检查您的配置并作出修改.");
        }
        // 对每一个connection 上配置的table 项进行解析
        List<String> expandedTables = TableExpandUtil.expandTableConf(DATABASE_TYPE, tables);
        if (null == expandedTables || expandedTables.isEmpty()) {
            throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR, "您配置的写入数据库表名称错误. DataX找不到您配置的表，请检查您的配置并作出修改.");
        }
        tableNum += expandedTables.size();
        originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.TABLE), expandedTables);
    }
    originalConfig.set(Constant.TABLE_NUMBER_MARK, tableNum);
}

Example 33 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class OdpsSplitUtil method splitPartitionedTable.

private static List<Configuration> splitPartitionedTable(Odps odps, Configuration originalConfig, int adviceNum) {
    List<Configuration> splittedConfigs = new ArrayList<Configuration>();
    List<String> partitions = originalConfig.getList(Key.PARTITION, String.class);
    if (null == partitions || partitions.isEmpty()) {
        throw DataXException.asDataXException(OdpsReaderErrorCode.ILLEGAL_VALUE, "您所配置的分区不能为空白.");
    }
    //splitMode 默认为 record
    String splitMode = originalConfig.getString(Key.SPLIT_MODE);
    Configuration tempConfig = null;
    if (partitions.size() > adviceNum || Constant.PARTITION_SPLIT_MODE.equals(splitMode)) {
        // 注意：此处没有把 sessionId 设置到 config 中去，所以后续在 task 中获取 sessionId 时，需要针对这种情况重新创建 sessionId
        for (String onePartition : partitions) {
            tempConfig = originalConfig.clone();
            tempConfig.set(Key.PARTITION, onePartition);
            splittedConfigs.add(tempConfig);
        }
        return splittedConfigs;
    } else {
        // 还需要计算对每个分区，切分份数等信息
        int eachPartitionShouldSplittedNumber = calculateEachPartitionShouldSplittedNumber(adviceNum, partitions.size());
        for (String onePartition : partitions) {
            List<Configuration> configs = splitOnePartition(odps, onePartition, eachPartitionShouldSplittedNumber, originalConfig);
            splittedConfigs.addAll(configs);
        }
        return splittedConfigs;
    }
}

Example 34 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class OtsWriterMasterProxy method split.

public List<Configuration> split(int mandatoryNumber) {
    LOG.info("Begin split and MandatoryNumber : {}", mandatoryNumber);
    List<Configuration> configurations = new ArrayList<Configuration>();
    for (int i = 0; i < mandatoryNumber; i++) {
        Configuration configuration = Configuration.newDefault();
        configuration.set(OTSConst.OTS_CONF, GsonParser.confToJson(this.conf));
        configurations.add(configuration);
    }
    LOG.info("End split.");
    assert (mandatoryNumber == configurations.size());
    return configurations;
}

Example 35 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class OriginalConfPretreatmentUtil method recognizeTableOrQuerySqlMode.

private static boolean recognizeTableOrQuerySqlMode(Configuration originalConfig) {
    List<Object> conns = originalConfig.getList(Constant.CONN_MARK, Object.class);
    List<Boolean> tableModeFlags = new ArrayList<Boolean>();
    List<Boolean> querySqlModeFlags = new ArrayList<Boolean>();
    String table = null;
    String querySql = null;
    boolean isTableMode = false;
    boolean isQuerySqlMode = false;
    for (int i = 0, len = conns.size(); i < len; i++) {
        Configuration connConf = Configuration.from(conns.get(i).toString());
        table = connConf.getString(Key.TABLE, null);
        querySql = connConf.getString(Key.QUERY_SQL, null);
        isTableMode = StringUtils.isNotBlank(table);
        tableModeFlags.add(isTableMode);
        isQuerySqlMode = StringUtils.isNotBlank(querySql);
        querySqlModeFlags.add(isQuerySqlMode);
        if (false == isTableMode && false == isQuerySqlMode) {
            // table 和 querySql 二者均未配制
            throw DataXException.asDataXException(DBUtilErrorCode.TABLE_QUERYSQL_MISSING, "您的配置有误. 因为table和querySql应该配置并且只能配置一个. 请检查您的配置并作出修改.");
        } else if (true == isTableMode && true == isQuerySqlMode) {
            // table 和 querySql 二者均配置
            throw DataXException.asDataXException(DBUtilErrorCode.TABLE_QUERYSQL_MIXED, "您的配置凌乱了. 因为datax不能同时既配置table又配置querySql.请检查您的配置并作出修改.");
        }
    }
    // 混合配制 table 和 querySql
    if (!ListUtil.checkIfValueSame(tableModeFlags) || !ListUtil.checkIfValueSame(tableModeFlags)) {
        throw DataXException.asDataXException(DBUtilErrorCode.TABLE_QUERYSQL_MIXED, "您配置凌乱了. 不能同时既配置table又配置querySql. 请检查您的配置并作出修改.");
    }
    return tableModeFlags.get(0);
}