Examples with Configuration com.alibaba.datax.common.util.Configuration used on opensource projects

Example 1 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class TransformerRegistry method loadTransformer.

public static void loadTransformer(String each) {
    String transformerPath = CoreConstant.DATAX_STORAGE_TRANSFORMER_HOME + File.separator + each;
    Configuration transformerConfiguration;
    try {
        transformerConfiguration = loadTransFormerConfig(transformerPath);
    } catch (Exception e) {
        LOG.error(String.format("skip transformer(%s),load transformer.json error, path = %s, ", each, transformerPath), e);
        return;
    }
    String className = transformerConfiguration.getString("class");
    if (StringUtils.isEmpty(className)) {
        LOG.error(String.format("skip transformer(%s),class not config, path = %s, config = %s", each, transformerPath, transformerConfiguration.beautify()));
        return;
    }
    String funName = transformerConfiguration.getString("name");
    if (!each.equals(funName)) {
        LOG.warn(String.format("transformer(%s) name not match transformer.json config name[%s], will ignore json's name, path = %s, config = %s", each, funName, transformerPath, transformerConfiguration.beautify()));
    }
    JarLoader jarLoader = new JarLoader(new String[] { transformerPath });
    try {
        Class<?> transformerClass = jarLoader.loadClass(className);
        Object transformer = transformerClass.newInstance();
        if (ComplexTransformer.class.isAssignableFrom(transformer.getClass())) {
            ((ComplexTransformer) transformer).setTransformerName(each);
            registComplexTransformer((ComplexTransformer) transformer, jarLoader, false);
        } else if (Transformer.class.isAssignableFrom(transformer.getClass())) {
            ((Transformer) transformer).setTransformerName(each);
            registTransformer((Transformer) transformer, jarLoader, false);
        } else {
            LOG.error(String.format("load Transformer class(%s) error, path = %s", className, transformerPath));
        }
    } catch (Exception e) {
        //错误funciton跳过
        LOG.error(String.format("skip transformer(%s),load Transformer class error, path = %s ", each, transformerPath), e);
    }
}

Example 2 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class ConfigParser method parseOnePluginConfig.

public static Configuration parseOnePluginConfig(final String path, final String type, Set<String> pluginSet, List<String> wantPluginNames) {
    String filePath = path + File.separator + "plugin.json";
    Configuration configuration = Configuration.from(new File(filePath));
    String pluginPath = configuration.getString("path");
    String pluginName = configuration.getString("name");
    if (!pluginSet.contains(pluginName)) {
        pluginSet.add(pluginName);
    } else {
        throw DataXException.asDataXException(FrameworkErrorCode.PLUGIN_INIT_ERROR, "插件加载失败,存在重复插件:" + filePath);
    }
    //不是想要的插件，返回null
    if (wantPluginNames != null && wantPluginNames.size() > 0 && !wantPluginNames.contains(pluginName)) {
        return null;
    }
    boolean isDefaultPath = StringUtils.isBlank(pluginPath);
    if (isDefaultPath) {
        configuration.set("path", path);
    }
    Configuration result = Configuration.newDefault();
    result.set(String.format("plugin.%s.%s", type, pluginName), configuration.getInternal());
    return result;
}

Example 3 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class ConfigParser method parseJobConfig.

public static Configuration parseJobConfig(final String path) {
    String jobContent = getJobContent(path);
    Configuration config = Configuration.from(jobContent);
    return SecretUtil.decryptSecretKey(config);
}

Example 4 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class ConfigParser method parsePluginConfig.

public static Configuration parsePluginConfig(List<String> wantPluginNames) {
    Configuration configuration = Configuration.newDefault();
    Set<String> replicaCheckPluginSet = new HashSet<String>();
    int complete = 0;
    for (final String each : ConfigParser.getDirAsList(CoreConstant.DATAX_PLUGIN_READER_HOME)) {
        Configuration eachReaderConfig = ConfigParser.parseOnePluginConfig(each, "reader", replicaCheckPluginSet, wantPluginNames);
        if (eachReaderConfig != null) {
            configuration.merge(eachReaderConfig, true);
            complete += 1;
        }
    }
    for (final String each : ConfigParser.getDirAsList(CoreConstant.DATAX_PLUGIN_WRITER_HOME)) {
        Configuration eachWriterConfig = ConfigParser.parseOnePluginConfig(each, "writer", replicaCheckPluginSet, wantPluginNames);
        if (eachWriterConfig != null) {
            configuration.merge(eachWriterConfig, true);
            complete += 1;
        }
    }
    if (wantPluginNames != null && wantPluginNames.size() > 0 && wantPluginNames.size() != complete) {
        throw DataXException.asDataXException(FrameworkErrorCode.PLUGIN_INIT_ERROR, "插件加载失败，未完成指定插件加载:" + wantPluginNames);
    }
    return configuration;
}

Example 5 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class ConfigParser method parse.

/**
     * 指定Job配置路径，ConfigParser会解析Job、Plugin、Core全部信息，并以Configuration返回
     */
public static Configuration parse(final String jobPath) {
    Configuration configuration = ConfigParser.parseJobConfig(jobPath);
    configuration.merge(ConfigParser.parseCoreConfig(CoreConstant.DATAX_CONF_PATH), false);
    // todo config优化，只捕获需要的plugin
    String readerPluginName = configuration.getString(CoreConstant.DATAX_JOB_CONTENT_READER_NAME);
    String writerPluginName = configuration.getString(CoreConstant.DATAX_JOB_CONTENT_WRITER_NAME);
    String preHandlerName = configuration.getString(CoreConstant.DATAX_JOB_PREHANDLER_PLUGINNAME);
    String postHandlerName = configuration.getString(CoreConstant.DATAX_JOB_POSTHANDLER_PLUGINNAME);
    Set<String> pluginList = new HashSet<String>();
    pluginList.add(readerPluginName);
    pluginList.add(writerPluginName);
    if (StringUtils.isNotEmpty(preHandlerName)) {
        pluginList.add(preHandlerName);
    }
    if (StringUtils.isNotEmpty(postHandlerName)) {
        pluginList.add(postHandlerName);
    }
    try {
        configuration.merge(parsePluginConfig(new ArrayList<String>(pluginList)), false);
    } catch (Exception e) {
        //吞掉异常，保持log干净。这里message足够。
        LOG.warn(String.format("插件[%s,%s]加载失败，1s后重试... Exception:%s ", readerPluginName, writerPluginName, e.getMessage()));
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e1) {
        //
        }
        configuration.merge(parsePluginConfig(new ArrayList<String>(pluginList)), false);
    }
    return configuration;
}