Examples with TaskGroupContainer com.alibaba.datax.core.taskgroup.TaskGroupContainer used on opensource projects

Example 1 with TaskGroupContainer

use of com.alibaba.datax.core.taskgroup.TaskGroupContainer in project DataX by alibaba.

the class TaskGroupContainerTest method testRealTransformer.

@Test
public void testRealTransformer() {
    LoadUtil.bind(configurationRealTransformer);
    this.configurationRealTransformer.set("plugin.writer.fakewriter.class", FakeOneReader.class.getName());
    this.configurationRealTransformer.set("plugin.writer.fakewriter.class", FakeLongTimeWriter.class.getName());
    this.configurationRealTransformer.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL, 1);
    Configuration jobContent = this.configurationRealTransformer.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).get(0);
    List<Configuration> jobContents = new ArrayList<Configuration>();
    jobContents.add(jobContent);
    this.configurationRealTransformer.set(CoreConstant.DATAX_JOB_CONTENT, jobContents);
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configurationRealTransformer);
    taskGroupContainer.start();
    Assert.assertTrue(State.SUCCEEDED == taskGroupContainer.getContainerCommunicator().collect().getState());
    Communication res = null;
    try {
        Method com = TaskGroupContainer.class.getDeclaredMethod("reportTaskGroupCommunication", Communication.class, int.class);
        com.setAccessible(true);
        res = (Communication) com.invoke(taskGroupContainer, new Communication(), 1);
        System.out.println("TaskGroup => " + CommunicationTool.Stringify.getSnapshot(res));
    } catch (Exception e) {
        e.printStackTrace();
    }
    Assert.assertTrue(res != null);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TOTAL_READ_RECORDS).longValue(), 30);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_SUCCEED_RECORDS).longValue(), 10);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FAILED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FILTER_RECORDS).longValue(), 10);
    Assert.assertTrue(res.getLongCounter(CommunicationTool.TRANSFORMER_USED_TIME).longValue() > 0);
}

Example 2 with TaskGroupContainer

use of com.alibaba.datax.core.taskgroup.TaskGroupContainer in project DataX by alibaba.

the class TaskGroupContainerTest method testWriterException.

@Test(expected = RuntimeException.class)
public void testWriterException() {
    this.configuration.set("plugin.writer.fakewriter.class", FakeExceptionWriter.class.getName());
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configuration);
    taskGroupContainer.start();
}

Example 3 with TaskGroupContainer

use of com.alibaba.datax.core.taskgroup.TaskGroupContainer in project DataX by alibaba.

the class Engine method start.

/* check job model (job/task) first */
public void start(Configuration allConf) {
    // 绑定column转换信息
    ColumnCast.bind(allConf);
    /**
         * 初始化PluginLoader，可以获取各种插件配置
         */
    LoadUtil.bind(allConf);
    boolean isJob = !("taskGroup".equalsIgnoreCase(allConf.getString(CoreConstant.DATAX_CORE_CONTAINER_MODEL)));
    //JobContainer会在schedule后再行进行设置和调整值
    int channelNumber = 0;
    AbstractContainer container;
    long instanceId;
    int taskGroupId = -1;
    if (isJob) {
        allConf.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_MODE, RUNTIME_MODE);
        container = new JobContainer(allConf);
        instanceId = allConf.getLong(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID, 0);
    } else {
        container = new TaskGroupContainer(allConf);
        instanceId = allConf.getLong(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID);
        taskGroupId = allConf.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID);
        channelNumber = allConf.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL);
    }
    //缺省打开perfTrace
    boolean traceEnable = allConf.getBool(CoreConstant.DATAX_CORE_CONTAINER_TRACE_ENABLE, true);
    boolean perfReportEnable = allConf.getBool(CoreConstant.DATAX_CORE_REPORT_DATAX_PERFLOG, true);
    //standlone模式的datax shell任务不进行汇报
    if (instanceId == -1) {
        perfReportEnable = false;
    }
    int priority = 0;
    try {
        priority = Integer.parseInt(System.getenv("SKYNET_PRIORITY"));
    } catch (NumberFormatException e) {
        LOG.warn("prioriy set to 0, because NumberFormatException, the value is: " + System.getProperty("PROIORY"));
    }
    Configuration jobInfoConfig = allConf.getConfiguration(CoreConstant.DATAX_JOB_JOBINFO);
    //初始化PerfTrace
    PerfTrace perfTrace = PerfTrace.getInstance(isJob, instanceId, taskGroupId, priority, traceEnable);
    perfTrace.setJobInfo(jobInfoConfig, perfReportEnable, channelNumber);
    container.start();
}

Example 4 with TaskGroupContainer

use of com.alibaba.datax.core.taskgroup.TaskGroupContainer in project DataX by alibaba.

the class TaskGroupContainerTest method testReaderException.

@Test(expected = RuntimeException.class)
public void testReaderException() {
    this.configuration.set("plugin.reader.fakereader.class", FakeExceptionReader.class.getCanonicalName());
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configuration);
    taskGroupContainer.start();
}

Example 5 with TaskGroupContainer

use of com.alibaba.datax.core.taskgroup.TaskGroupContainer in project DataX by alibaba.

the class TaskGroupContainerTest method testLongTimeWriter.

@Test
public void testLongTimeWriter() {
    this.configuration.set("plugin.writer.fakewriter.class", FakeOneReader.class.getName());
    this.configuration.set("plugin.writer.fakewriter.class", FakeLongTimeWriter.class.getName());
    this.configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL, 1);
    Configuration jobContent = this.configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).get(0);
    List<Configuration> jobContents = new ArrayList<Configuration>();
    jobContents.add(jobContent);
    this.configuration.set(CoreConstant.DATAX_JOB_CONTENT, jobContents);
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configuration);
    taskGroupContainer.start();
    Assert.assertTrue(State.SUCCEEDED == taskGroupContainer.getContainerCommunicator().collect().getState());
    Communication res = null;
    try {
        Method com = TaskGroupContainer.class.getDeclaredMethod("reportTaskGroupCommunication", Communication.class, int.class);
        com.setAccessible(true);
        res = (Communication) com.invoke(taskGroupContainer, new Communication(), 1);
        System.out.println("TaskGroup => " + CommunicationTool.Stringify.getSnapshot(res));
    } catch (Exception e) {
        e.printStackTrace();
    }
    Assert.assertTrue(res != null);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TOTAL_READ_RECORDS).longValue(), 40);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_SUCCEED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FAILED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FILTER_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_USED_TIME).longValue(), 0);
    System.out.println(res.getMessage());
}