Examples with Configuration com.alibaba.datax.common.util.Configuration used on opensource projects

Example 66 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class TransformerUtil method buildTransformerInfo.

public static List<TransformerExecution> buildTransformerInfo(Configuration taskConfig) {
    List<Configuration> tfConfigs = taskConfig.getListConfiguration(CoreConstant.JOB_TRANSFORMER);
    if (tfConfigs == null || tfConfigs.size() == 0) {
        return null;
    }
    List<TransformerExecution> result = new ArrayList<TransformerExecution>();
    List<String> functionNames = new ArrayList<String>();
    for (Configuration configuration : tfConfigs) {
        String functionName = configuration.getString("name");
        if (StringUtils.isEmpty(functionName)) {
            throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_CONFIGURATION_ERROR, "config=" + configuration.toJSON());
        }
        if (functionName.equals("dx_groovy") && functionNames.contains("dx_groovy")) {
            throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_CONFIGURATION_ERROR, "dx_groovy can be invoke once only.");
        }
        functionNames.add(functionName);
    }
    /**
         * 延迟load 第三方插件的function，并按需load
         */
    LOG.info(String.format(" user config tranformers [%s], loading...", functionNames));
    TransformerRegistry.loadTransformerFromLocalStorage(functionNames);
    int i = 0;
    for (Configuration configuration : tfConfigs) {
        String functionName = configuration.getString("name");
        TransformerInfo transformerInfo = TransformerRegistry.getTransformer(functionName);
        if (transformerInfo == null) {
            throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_NOTFOUND_ERROR, "name=" + functionName);
        }
        /**
             * 具体的UDF对应一个paras
             */
        TransformerExecutionParas transformerExecutionParas = new TransformerExecutionParas();
        /**
             * groovy function仅仅只有code
             */
        if (!functionName.equals("dx_groovy") && !functionName.equals("dx_fackGroovy")) {
            Integer columnIndex = configuration.getInt(CoreConstant.TRANSFORMER_PARAMETER_COLUMNINDEX);
            if (columnIndex == null) {
                throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, "columnIndex must be set by UDF:name=" + functionName);
            }
            transformerExecutionParas.setColumnIndex(columnIndex);
            List<String> paras = configuration.getList(CoreConstant.TRANSFORMER_PARAMETER_PARAS, String.class);
            if (paras != null && paras.size() > 0) {
                transformerExecutionParas.setParas(paras.toArray(new String[0]));
            }
        } else {
            String code = configuration.getString(CoreConstant.TRANSFORMER_PARAMETER_CODE);
            if (StringUtils.isEmpty(code)) {
                throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, "groovy code must be set by UDF:name=" + functionName);
            }
            transformerExecutionParas.setCode(code);
            List<String> extraPackage = configuration.getList(CoreConstant.TRANSFORMER_PARAMETER_EXTRAPACKAGE, String.class);
            if (extraPackage != null && extraPackage.size() > 0) {
                transformerExecutionParas.setExtraPackage(extraPackage);
            }
        }
        transformerExecutionParas.settContext(configuration.getMap(CoreConstant.TRANSFORMER_PARAMETER_CONTEXT));
        TransformerExecution transformerExecution = new TransformerExecution(transformerInfo, transformerExecutionParas);
        transformerExecution.genFinalParas();
        result.add(transformerExecution);
        i++;
        LOG.info(String.format(" %s of transformer init success. name=%s, isNative=%s parameter = %s", i, transformerInfo.getTransformer().getTransformerName(), transformerInfo.isNative(), configuration.getConfiguration("parameter")));
    }
    return result;
}

Example 67 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class LoadUtil method getJarLoader.

public static synchronized JarLoader getJarLoader(PluginType pluginType, String pluginName) {
    Configuration pluginConf = getPluginConf(pluginType, pluginName);
    JarLoader jarLoader = jarLoaderCenter.get(generatePluginKey(pluginType, pluginName));
    if (null == jarLoader) {
        String pluginPath = pluginConf.getString("path");
        if (StringUtils.isBlank(pluginPath)) {
            throw DataXException.asDataXException(FrameworkErrorCode.RUNTIME_ERROR, String.format("%s插件[%s]路径非法!", pluginType, pluginName));
        }
        jarLoader = new JarLoader(new String[] { pluginPath });
        jarLoaderCenter.put(generatePluginKey(pluginType, pluginName), jarLoader);
    }
    return jarLoader;
}

Example 68 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class DrdsReaderSplitUtil method doDrdsReaderSplit.

private static List<Configuration> doDrdsReaderSplit(Configuration originalSliceConfig) {
    List<Configuration> splittedConfigurations = new ArrayList<Configuration>();
    Map<String, List<String>> topology = getTopology(originalSliceConfig);
    if (null == topology || topology.isEmpty()) {
        throw DataXException.asDataXException(DrdsReaderErrorCode.GET_TOPOLOGY_FAILED, "获取 drds 表拓扑结构失败, 拓扑结构不能为空.");
    } else {
        String table = originalSliceConfig.getString(Key.TABLE).trim();
        String column = originalSliceConfig.getString(Key.COLUMN).trim();
        String where = originalSliceConfig.getString(Key.WHERE, null);
        // 不能带英语分号结尾
        String sql = SingleTableSplitUtil.buildQuerySql(column, table, where);
        // 根据拓扑拆分任务
        for (Map.Entry<String, List<String>> entry : topology.entrySet()) {
            String group = entry.getKey();
            StringBuilder sqlbuilder = new StringBuilder();
            sqlbuilder.append("/*+TDDL({'extra':{'MERGE_UNION':'false'},'type':'direct',");
            sqlbuilder.append("'vtab':'").append(table).append("',");
            sqlbuilder.append("'dbid':'").append(group).append("',");
            sqlbuilder.append("'realtabs':[");
            Iterator<String> it = entry.getValue().iterator();
            while (it.hasNext()) {
                String realTable = it.next();
                sqlbuilder.append('\'').append(realTable).append('\'');
                if (it.hasNext()) {
                    sqlbuilder.append(',');
                }
            }
            sqlbuilder.append("]})*/");
            sqlbuilder.append(sql);
            Configuration param = originalSliceConfig.clone();
            param.set(Key.QUERY_SQL, sqlbuilder.toString());
            splittedConfigurations.add(param);
        }
        return splittedConfigurations;
    }
}

Example 69 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class ConfigParserTest method test.

@Test
public void test() throws URISyntaxException {
    Configuration configuration = ConfigParser.parse(jobPath);
    System.out.println(configuration.toJSON());
    Assert.assertTrue(configuration.getList("job.content").size() == 2);
    Assert.assertTrue(configuration.getString("job.content[0].reader.name").equals("fakereader"));
    Assert.assertTrue(configuration.getString("job.content[1].reader.name").equals("fakereader"));
    Assert.assertTrue(configuration.getString("job.content[0].writer.name").equals("fakewriter"));
    Assert.assertTrue(configuration.getString("job.content[1].writer.name").equals("fakewriter"));
    System.out.println(configuration.getConfiguration("plugin").toJSON());
    configuration = configuration.getConfiguration("plugin");
    Assert.assertTrue(configuration.getString("reader.fakereader.name").equals("fakereader"));
    Assert.assertTrue(configuration.getString("writer.fakewriter.name").equals("fakewriter"));
}

Example 70 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class ConfigParserTest method secretTest.

@Test
public void secretTest() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
    String password = "password";
    String accessKey = "accessKey";
    String readerParamPath = "job.content[0].reader.parameter";
    String writerParamPath = "job.content[1].writer.parameter";
    Map<String, String> secretMap = getPublicKeyMap();
    String keyVersion = null;
    for (String version : secretMap.keySet()) {
        keyVersion = version;
        break;
    }
    Configuration config = ConfigParser.parse(jobPath);
    config.set(CoreConstant.DATAX_JOB_SETTING_KEYVERSION, keyVersion);
    config.set(readerParamPath + ".*password", SecretUtil.encrypt(password, secretMap.get(keyVersion), SecretUtil.KEY_ALGORITHM_RSA));
    config.set(readerParamPath + ".*long", 100);
    config.set(writerParamPath + ".*accessKey", SecretUtil.encrypt(accessKey, secretMap.get(keyVersion), SecretUtil.KEY_ALGORITHM_RSA));
    config.set(writerParamPath + ".*long", 200);
    config = SecretUtil.decryptSecretKey(config);
    Assert.assertTrue(password.equals(config.getString(readerParamPath + ".password")));
    Assert.assertTrue(config.isSecretPath(readerParamPath + ".password"));
    Assert.assertTrue(config.get(readerParamPath + ".*long") != null);
    Assert.assertTrue(accessKey.equals(config.getString(writerParamPath + ".accessKey")));
    Assert.assertTrue(config.isSecretPath(writerParamPath + ".accessKey"));
    Assert.assertTrue(config.get(writerParamPath + ".*long") != null);
    Assert.assertTrue(StringUtils.isBlank(config.getString(readerParamPath + ".*password")));
    Assert.assertTrue(StringUtils.isBlank(config.getString(writerParamPath + ".*accessKey")));
}