Examples with Configuration com.alibaba.datax.common.util.Configuration used on opensource projects

Example 41 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class DBUtil method loadDriverClass.

public static void loadDriverClass(String pluginType, String pluginName) {
    try {
        String pluginJsonPath = StringUtils.join(new String[] { System.getProperty("datax.home"), "plugin", pluginType, String.format("%s%s", pluginName, pluginType), "plugin.json" }, File.separator);
        Configuration configuration = Configuration.from(new File(pluginJsonPath));
        List<String> drivers = configuration.getList("drivers", String.class);
        for (String driver : drivers) {
            Class.forName(driver);
        }
    } catch (ClassNotFoundException e) {
        throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR, "数据库驱动加载错误, 请确认libs目录有驱动jar包且plugin.json中drivers配置驱动类正确!", e);
    }
}

Example 42 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class WriterUtil method preCheckPostSQL.

public static void preCheckPostSQL(Configuration originalConfig, DataBaseType type) {
    List<Object> conns = originalConfig.getList(Constant.CONN_MARK, Object.class);
    Configuration connConf = Configuration.from(conns.get(0).toString());
    String table = connConf.getList(Key.TABLE, String.class).get(0);
    List<String> postSqls = originalConfig.getList(Key.POST_SQL, String.class);
    List<String> renderedPostSqls = WriterUtil.renderPreOrPostSqls(postSqls, table);
    if (null != renderedPostSqls && !renderedPostSqls.isEmpty()) {
        LOG.info("Begin to preCheck postSqls:[{}].", StringUtils.join(renderedPostSqls, ";"));
        for (String sql : renderedPostSqls) {
            try {
                DBUtil.sqlValid(sql, type);
            } catch (ParserException e) {
                throw RdbmsException.asPostSQLParserException(type, e, sql);
            }
        }
    }
}

Example 43 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class WriterUtil method doSplit.

//TODO 切分报错
public static List<Configuration> doSplit(Configuration simplifiedConf, int adviceNumber) {
    List<Configuration> splitResultConfigs = new ArrayList<Configuration>();
    int tableNumber = simplifiedConf.getInt(Constant.TABLE_NUMBER_MARK);
    //处理单表的情况
    if (tableNumber == 1) {
        //由于在之前的  master prepare 中已经把 table,jdbcUrl 提取出来，所以这里处理十分简单
        for (int j = 0; j < adviceNumber; j++) {
            splitResultConfigs.add(simplifiedConf.clone());
        }
        return splitResultConfigs;
    }
    if (tableNumber != adviceNumber) {
        throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR, String.format("您的配置文件中的列配置信息有误. 您要写入的目的端的表个数是:%s , 但是根据系统建议需要切分的份数是：%s. 请检查您的配置并作出修改.", tableNumber, adviceNumber));
    }
    String jdbcUrl;
    List<String> preSqls = simplifiedConf.getList(Key.PRE_SQL, String.class);
    List<String> postSqls = simplifiedConf.getList(Key.POST_SQL, String.class);
    List<Object> conns = simplifiedConf.getList(Constant.CONN_MARK, Object.class);
    for (Object conn : conns) {
        Configuration sliceConfig = simplifiedConf.clone();
        Configuration connConf = Configuration.from(conn.toString());
        jdbcUrl = connConf.getString(Key.JDBC_URL);
        sliceConfig.set(Key.JDBC_URL, jdbcUrl);
        sliceConfig.remove(Constant.CONN_MARK);
        List<String> tables = connConf.getList(Key.TABLE, String.class);
        for (String table : tables) {
            Configuration tempSlice = sliceConfig.clone();
            tempSlice.set(Key.TABLE, table);
            tempSlice.set(Key.PRE_SQL, renderPreOrPostSqls(preSqls, table));
            tempSlice.set(Key.POST_SQL, renderPreOrPostSqls(postSqls, table));
            splitResultConfigs.add(tempSlice);
        }
    }
    return splitResultConfigs;
}

Example 44 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class WriterUtil method preCheckPrePareSQL.

public static void preCheckPrePareSQL(Configuration originalConfig, DataBaseType type) {
    List<Object> conns = originalConfig.getList(Constant.CONN_MARK, Object.class);
    Configuration connConf = Configuration.from(conns.get(0).toString());
    String table = connConf.getList(Key.TABLE, String.class).get(0);
    List<String> preSqls = originalConfig.getList(Key.PRE_SQL, String.class);
    List<String> renderedPreSqls = WriterUtil.renderPreOrPostSqls(preSqls, table);
    if (null != renderedPreSqls && !renderedPreSqls.isEmpty()) {
        LOG.info("Begin to preCheck preSqls:[{}].", StringUtils.join(renderedPreSqls, ";"));
        for (String sql : renderedPreSqls) {
            try {
                DBUtil.sqlValid(sql, type);
            } catch (ParserException e) {
                throw RdbmsException.asPreSQLParserException(type, e, sql);
            }
        }
    }
}

Example 45 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class OdpsSplitUtil method splitForNonPartitionedTable.

private static List<Configuration> splitForNonPartitionedTable(Odps odps, int adviceNum, Configuration sliceConfig) {
    List<Configuration> params = new ArrayList<Configuration>();
    String tunnelServer = sliceConfig.getString(Key.TUNNEL_SERVER);
    String tableName = sliceConfig.getString(Key.TABLE);
    String projectName = sliceConfig.getString(Key.PROJECT);
    DownloadSession session = OdpsUtil.createMasterSessionForNonPartitionedTable(odps, tunnelServer, projectName, tableName);
    String id = session.getId();
    long count = session.getRecordCount();
    List<Pair<Long, Long>> splitResult = splitRecordCount(count, adviceNum);
    for (Pair<Long, Long> pair : splitResult) {
        Configuration iParam = sliceConfig.clone();
        iParam.set(Constant.SESSION_ID, id);
        iParam.set(Constant.START_INDEX, pair.getLeft().longValue());
        iParam.set(Constant.STEP_COUNT, pair.getRight().longValue());
        params.add(iParam);
    }
    return params;
}