Examples with Configuration com.alibaba.datax.common.util.Configuration used on opensource projects

Example 21 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class LoadUtil method loadPluginClass.

/**
     * 反射出具体plugin实例
     *
     * @param pluginType
     * @param pluginName
     * @param pluginRunType
     * @return
     */
@SuppressWarnings("unchecked")
private static synchronized Class<? extends AbstractPlugin> loadPluginClass(PluginType pluginType, String pluginName, ContainerType pluginRunType) {
    Configuration pluginConf = getPluginConf(pluginType, pluginName);
    JarLoader jarLoader = LoadUtil.getJarLoader(pluginType, pluginName);
    try {
        return (Class<? extends AbstractPlugin>) jarLoader.loadClass(pluginConf.getString("class") + "$" + pluginRunType.value());
    } catch (Exception e) {
        throw DataXException.asDataXException(FrameworkErrorCode.RUNTIME_ERROR, e);
    }
}

Example 22 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class JobAssignUtilTest method test_02.

@Test
public void test_02() {
    String jobString = "{\"job\":{\"setting\":{},\"content\":[{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}},{\"reader\":{\"type\":\"fakereader\",\"parameter\":{}},\"writer\":{\"type\":\"fakewriter\",\"parameter\":{}}}]}}";
    Configuration configuration = Configuration.from(jobString);
    int taskNumber = StringUtils.countMatches(jobString, "fakereader");
    System.out.println("taskNumber:" + taskNumber);
    for (int i = 0; i < taskNumber; i++) {
        configuration.set("job.content[" + i + "].taskId", i);
    }
    //        System.out.println(configuration.beautify());
    int channelNumber = 13;
    int channelsPerTaskGroup = 5;
    List<Configuration> result = JobAssignUtil.assignFairly(configuration, channelNumber, channelsPerTaskGroup);
    System.out.println("===================================");
    for (Configuration conf : result) {
        System.out.println(conf.beautify());
        System.out.println("----------------");
    }
//        System.out.println(configuration);
}

Example 23 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class JobAssignUtilTest method test_01.

@Test
public void test_01() {
    Configuration configuration = Configuration.from(new File(JobAssignUtil.class.getResource("/job/job.json").getFile()));
    configuration.set("job.content[0].taskId", 0);
    configuration.set("job.content[1].taskId", 1);
    System.out.println(configuration.beautify());
    int channelNumber = 3;
    int channelsPerTaskGroup = 1;
    List<Configuration> result = JobAssignUtil.assignFairly(configuration, channelNumber, channelsPerTaskGroup);
    System.out.println("===================================");
    for (Configuration conf : result) {
        System.out.println(conf.beautify());
        System.out.println("----------------");
    }
    System.out.println(configuration);
}

Example 24 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class JobContainerTest method testPreHandlerByReader.

@Test
public void testPreHandlerByReader() throws Exception {
    Configuration copyConfig = this.configuration.clone();
    copyConfig.set(CoreConstant.DATAX_JOB_PREHANDLER_PLUGINTYPE, "reader");
    copyConfig.set(CoreConstant.DATAX_JOB_PREHANDLER_PLUGINNAME, "fakereader");
    JobContainer jobContainer = new JobContainer(copyConfig);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("preHandle");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    System.out.println(copyConfig.get("job.preHandler.test"));
    Assert.assertEquals("readPreDone", copyConfig.get("job.preHandler.test"));
}

Example 25 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class JobContainerTest method testMergeReaderAndWriterSlicesConfigsException.

@Test(expected = Exception.class)
public void testMergeReaderAndWriterSlicesConfigsException() throws Exception {
    JobContainer jobContainer = new JobContainer(this.configuration);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("init");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
    int readerSplitNumber = 100;
    int writerSplitNumber = readerSplitNumber + 1;
    List<Configuration> readerSplitConfigurations = new ArrayList<Configuration>();
    List<Configuration> writerSplitConfigurations = new ArrayList<Configuration>();
    for (int i = 0; i < readerSplitNumber; i++) {
        Configuration readerOneConfig = Configuration.newDefault();
        readerSplitConfigurations.add(readerOneConfig);
    }
    for (int i = 0; i < writerSplitNumber; i++) {
        Configuration readerOneConfig = Configuration.newDefault();
        writerSplitConfigurations.add(readerOneConfig);
    }
    initMethod = jobContainer.getClass().getDeclaredMethod("mergeReaderAndWriterSlicesConfigs", List.class, List.class);
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, readerSplitConfigurations, writerSplitConfigurations);
}