Examples with JobContainer com.alibaba.datax.core.job.JobContainer used on opensource projects

Example 11 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class JobContainerTest method testMergeReaderAndWriterSlicesConfigs.

@SuppressWarnings("unchecked")
@Test
public void testMergeReaderAndWriterSlicesConfigs() throws Exception {
    JobContainer jobContainer = new JobContainer(this.configuration);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("init");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
    int splitNumber = 100;
    List<Configuration> readerSplitConfigurations = new ArrayList<Configuration>();
    List<Configuration> writerSplitConfigurations = new ArrayList<Configuration>();
    for (int i = 0; i < splitNumber; i++) {
        Configuration readerOneConfig = Configuration.newDefault();
        List<String> jdbcUrlArray = new ArrayList<String>();
        jdbcUrlArray.add(String.format("jdbc:mysql://localhost:3305/db%04d", i));
        readerOneConfig.set("jdbcUrl", jdbcUrlArray);
        List<String> tableArray = new ArrayList<String>();
        tableArray.add(String.format("jingxing_%04d", i));
        readerOneConfig.set("table", tableArray);
        readerSplitConfigurations.add(readerOneConfig);
        Configuration writerOneConfig = Configuration.newDefault();
        List<String> odpsUrlArray = new ArrayList<String>();
        odpsUrlArray.add(String.format("odps://localhost:3305/db%04d", i));
        writerOneConfig.set("jdbcUrl", odpsUrlArray);
        List<String> odpsTableArray = new ArrayList<String>();
        odpsTableArray.add(String.format("jingxing_%04d", i));
        writerOneConfig.set("table", odpsTableArray);
        writerSplitConfigurations.add(writerOneConfig);
    }
    initMethod = jobContainer.getClass().getDeclaredMethod("mergeReaderAndWriterTaskConfigs", List.class, List.class);
    initMethod.setAccessible(true);
    List<Configuration> mergedConfigs = (List<Configuration>) initMethod.invoke(jobContainer, readerSplitConfigurations, writerSplitConfigurations);
    Assert.assertEquals("merge number equals to split number", splitNumber, mergedConfigs.size());
    for (Configuration sliceConfig : mergedConfigs) {
        Assert.assertNotNull("reader name not null", sliceConfig.getString(CoreConstant.JOB_READER_NAME));
        Assert.assertNotNull("reader name not null", sliceConfig.getString(CoreConstant.JOB_READER_PARAMETER));
        Assert.assertNotNull("reader name not null", sliceConfig.getString(CoreConstant.JOB_WRITER_NAME));
        Assert.assertNotNull("reader name not null", sliceConfig.getString(CoreConstant.JOB_WRITER_PARAMETER));
        Assert.assertTrue("has slice id", sliceConfig.getInt(CoreConstant.TASK_ID) >= 0);
    }
}

Example 12 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class JobContainerTest method testStart.

/**
     * standalone模式下点对点跑完全部流程
     */
@Test
public void testStart() {
    JobContainer jobContainer = new JobContainer(this.configuration);
    jobContainer.start();
}

Example 13 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class JobContainerTest method distributeTasksToTaskGroupContainerTest.

/**
     * 分发测试函数，可根据不同的通道数、每个taskGroup平均包括的channel数得到最优的分发结果
     * 注意：默认的tasks是采用faker里切分出的1024个tasks
     *
     * @param channelNumber
     * @param channelsPerTaskGroupContainer
     * @throws Exception
     */
@SuppressWarnings("unchecked")
private void distributeTasksToTaskGroupContainerTest(int channelNumber, int channelsPerTaskGroupContainer) throws Exception {
    JobContainer jobContainer = new JobContainer(this.configuration);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("init");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
    initMethod = jobContainer.getClass().getDeclaredMethod("split");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
    int tasksNumber = this.configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).size();
    int averSlicesPerChannel = tasksNumber / channelNumber;
    initMethod = jobContainer.getClass().getDeclaredMethod("distributeTasksToTaskGroup", int.class, int.class, int.class);
    initMethod.setAccessible(true);
    List<Configuration> taskGroupConfigs = (List<Configuration>) initMethod.invoke(jobContainer, averSlicesPerChannel, channelNumber, channelsPerTaskGroupContainer);
    initMethod.setAccessible(false);
    Assert.assertEquals("task size check", channelNumber / channelsPerTaskGroupContainer + (channelNumber % channelsPerTaskGroupContainer > 0 ? 1 : 0), taskGroupConfigs.size());
    int sumSlices = 0;
    for (Configuration taskGroupConfig : taskGroupConfigs) {
        Assert.assertNotNull("have set taskGroupId", taskGroupConfig.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID));
        int channelNo = taskGroupConfig.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL);
        Assert.assertNotNull("have set task channel number", channelNo);
        int taskNumber = taskGroupConfig.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).size();
        sumSlices += taskNumber;
        Assert.assertTrue("task has average tasks", taskNumber / channelNo == averSlicesPerChannel);
    }
    Assert.assertEquals("slices equal to split sum", tasksNumber, sumSlices);
}

Example 14 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class JobContainerTest method testPostHandler.

@Test
public void testPostHandler() throws Exception {
    JobContainer jobContainer = new JobContainer(this.configuration);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("postHandle");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    System.out.println(this.configuration.get("job.postHandler.test"));
    Assert.assertEquals("writePostDone", this.configuration.get("job.postHandler.test"));
}