Examples with Configuration com.alibaba.datax.common.util.Configuration used on opensource projects

Example 71 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class JobContainerTest method testMergeReaderAndWriterSlicesConfigs.

@SuppressWarnings("unchecked")
@Test
public void testMergeReaderAndWriterSlicesConfigs() throws Exception {
    JobContainer jobContainer = new JobContainer(this.configuration);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("init");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
    int splitNumber = 100;
    List<Configuration> readerSplitConfigurations = new ArrayList<Configuration>();
    List<Configuration> writerSplitConfigurations = new ArrayList<Configuration>();
    for (int i = 0; i < splitNumber; i++) {
        Configuration readerOneConfig = Configuration.newDefault();
        List<String> jdbcUrlArray = new ArrayList<String>();
        jdbcUrlArray.add(String.format("jdbc:mysql://localhost:3305/db%04d", i));
        readerOneConfig.set("jdbcUrl", jdbcUrlArray);
        List<String> tableArray = new ArrayList<String>();
        tableArray.add(String.format("jingxing_%04d", i));
        readerOneConfig.set("table", tableArray);
        readerSplitConfigurations.add(readerOneConfig);
        Configuration writerOneConfig = Configuration.newDefault();
        List<String> odpsUrlArray = new ArrayList<String>();
        odpsUrlArray.add(String.format("odps://localhost:3305/db%04d", i));
        writerOneConfig.set("jdbcUrl", odpsUrlArray);
        List<String> odpsTableArray = new ArrayList<String>();
        odpsTableArray.add(String.format("jingxing_%04d", i));
        writerOneConfig.set("table", odpsTableArray);
        writerSplitConfigurations.add(writerOneConfig);
    }
    initMethod = jobContainer.getClass().getDeclaredMethod("mergeReaderAndWriterTaskConfigs", List.class, List.class);
    initMethod.setAccessible(true);
    List<Configuration> mergedConfigs = (List<Configuration>) initMethod.invoke(jobContainer, readerSplitConfigurations, writerSplitConfigurations);
    Assert.assertEquals("merge number equals to split number", splitNumber, mergedConfigs.size());
    for (Configuration sliceConfig : mergedConfigs) {
        Assert.assertNotNull("reader name not null", sliceConfig.getString(CoreConstant.JOB_READER_NAME));
        Assert.assertNotNull("reader name not null", sliceConfig.getString(CoreConstant.JOB_READER_PARAMETER));
        Assert.assertNotNull("reader name not null", sliceConfig.getString(CoreConstant.JOB_WRITER_NAME));
        Assert.assertNotNull("reader name not null", sliceConfig.getString(CoreConstant.JOB_WRITER_PARAMETER));
        Assert.assertTrue("has slice id", sliceConfig.getInt(CoreConstant.TASK_ID) >= 0);
    }
}

Example 72 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class JobContainerTest method distributeTasksToTaskGroupContainerTest.

/**
     * 分发测试函数，可根据不同的通道数、每个taskGroup平均包括的channel数得到最优的分发结果
     * 注意：默认的tasks是采用faker里切分出的1024个tasks
     *
     * @param channelNumber
     * @param channelsPerTaskGroupContainer
     * @throws Exception
     */
@SuppressWarnings("unchecked")
private void distributeTasksToTaskGroupContainerTest(int channelNumber, int channelsPerTaskGroupContainer) throws Exception {
    JobContainer jobContainer = new JobContainer(this.configuration);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("init");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
    initMethod = jobContainer.getClass().getDeclaredMethod("split");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
    int tasksNumber = this.configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).size();
    int averSlicesPerChannel = tasksNumber / channelNumber;
    initMethod = jobContainer.getClass().getDeclaredMethod("distributeTasksToTaskGroup", int.class, int.class, int.class);
    initMethod.setAccessible(true);
    List<Configuration> taskGroupConfigs = (List<Configuration>) initMethod.invoke(jobContainer, averSlicesPerChannel, channelNumber, channelsPerTaskGroupContainer);
    initMethod.setAccessible(false);
    Assert.assertEquals("task size check", channelNumber / channelsPerTaskGroupContainer + (channelNumber % channelsPerTaskGroupContainer > 0 ? 1 : 0), taskGroupConfigs.size());
    int sumSlices = 0;
    for (Configuration taskGroupConfig : taskGroupConfigs) {
        Assert.assertNotNull("have set taskGroupId", taskGroupConfig.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID));
        int channelNo = taskGroupConfig.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL);
        Assert.assertNotNull("have set task channel number", channelNo);
        int taskNumber = taskGroupConfig.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).size();
        sumSlices += taskNumber;
        Assert.assertTrue("task has average tasks", taskNumber / channelNo == averSlicesPerChannel);
    }
    Assert.assertEquals("slices equal to split sum", tasksNumber, sumSlices);
}

Example 73 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class TaskGroupContainerTest method testLongTimeWriter.

@Test
public void testLongTimeWriter() {
    this.configuration.set("plugin.writer.fakewriter.class", FakeOneReader.class.getName());
    this.configuration.set("plugin.writer.fakewriter.class", FakeLongTimeWriter.class.getName());
    this.configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL, 1);
    Configuration jobContent = this.configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).get(0);
    List<Configuration> jobContents = new ArrayList<Configuration>();
    jobContents.add(jobContent);
    this.configuration.set(CoreConstant.DATAX_JOB_CONTENT, jobContents);
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configuration);
    taskGroupContainer.start();
    Assert.assertTrue(State.SUCCEEDED == taskGroupContainer.getContainerCommunicator().collect().getState());
    Communication res = null;
    try {
        Method com = TaskGroupContainer.class.getDeclaredMethod("reportTaskGroupCommunication", Communication.class, int.class);
        com.setAccessible(true);
        res = (Communication) com.invoke(taskGroupContainer, new Communication(), 1);
        System.out.println("TaskGroup => " + CommunicationTool.Stringify.getSnapshot(res));
    } catch (Exception e) {
        e.printStackTrace();
    }
    Assert.assertTrue(res != null);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TOTAL_READ_RECORDS).longValue(), 40);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_SUCCEED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FAILED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FILTER_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_USED_TIME).longValue(), 0);
    System.out.println(res.getMessage());
}

Example 74 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class TaskGroupContainerTest method testFakeTransformer.

@Test
public void testFakeTransformer() {
    TransformerRegistry.registTransformer(new FakeSubstrTransformer());
    TransformerRegistry.registTransformer(new FakeReplaceTransformer());
    TransformerRegistry.registTransformer(new FakeGroovyTransformer());
    LoadUtil.bind(configurationFakeTransformer);
    this.configurationFakeTransformer.set("plugin.writer.fakewriter.class", FakeOneReader.class.getName());
    this.configurationFakeTransformer.set("plugin.writer.fakewriter.class", FakeLongTimeWriter.class.getName());
    this.configurationFakeTransformer.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL, 1);
    Configuration jobContent = this.configurationFakeTransformer.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).get(0);
    List<Configuration> jobContents = new ArrayList<Configuration>();
    jobContents.add(jobContent);
    this.configurationFakeTransformer.set(CoreConstant.DATAX_JOB_CONTENT, jobContents);
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configurationFakeTransformer);
    taskGroupContainer.start();
    Assert.assertTrue(State.SUCCEEDED == taskGroupContainer.getContainerCommunicator().collect().getState());
    Communication res = null;
    try {
        Method com = TaskGroupContainer.class.getDeclaredMethod("reportTaskGroupCommunication", Communication.class, int.class);
        com.setAccessible(true);
        res = (Communication) com.invoke(taskGroupContainer, new Communication(), 1);
        System.out.println("TaskGroup => " + CommunicationTool.Stringify.getSnapshot(res));
    } catch (Exception e) {
        e.printStackTrace();
    }
    Assert.assertTrue(res != null);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TOTAL_READ_RECORDS).longValue(), 30);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_SUCCEED_RECORDS).longValue(), 10);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FAILED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FILTER_RECORDS).longValue(), 10);
}

Example 75 with Configuration

use of com.alibaba.datax.common.util.Configuration in project DataX by alibaba.

the class Consumer method before.

@Before
public void before() {
    System.out.println(ConfigurationProducer.produce().toJSON());
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 0);
    this.channel = new MemoryChannel(configuration);
    this.channel.setCommunication(new Communication());
}