Examples with JobContainer com.alibaba.datax.core.job.JobContainer used on opensource projects

Example 6 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class Engine method start.

/* check job model (job/task) first */
public void start(Configuration allConf) {
    // 绑定column转换信息
    ColumnCast.bind(allConf);
    /**
         * 初始化PluginLoader，可以获取各种插件配置
         */
    LoadUtil.bind(allConf);
    boolean isJob = !("taskGroup".equalsIgnoreCase(allConf.getString(CoreConstant.DATAX_CORE_CONTAINER_MODEL)));
    //JobContainer会在schedule后再行进行设置和调整值
    int channelNumber = 0;
    AbstractContainer container;
    long instanceId;
    int taskGroupId = -1;
    if (isJob) {
        allConf.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_MODE, RUNTIME_MODE);
        container = new JobContainer(allConf);
        instanceId = allConf.getLong(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID, 0);
    } else {
        container = new TaskGroupContainer(allConf);
        instanceId = allConf.getLong(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID);
        taskGroupId = allConf.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID);
        channelNumber = allConf.getInt(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL);
    }
    //缺省打开perfTrace
    boolean traceEnable = allConf.getBool(CoreConstant.DATAX_CORE_CONTAINER_TRACE_ENABLE, true);
    boolean perfReportEnable = allConf.getBool(CoreConstant.DATAX_CORE_REPORT_DATAX_PERFLOG, true);
    //standlone模式的datax shell任务不进行汇报
    if (instanceId == -1) {
        perfReportEnable = false;
    }
    int priority = 0;
    try {
        priority = Integer.parseInt(System.getenv("SKYNET_PRIORITY"));
    } catch (NumberFormatException e) {
        LOG.warn("prioriy set to 0, because NumberFormatException, the value is: " + System.getProperty("PROIORY"));
    }
    Configuration jobInfoConfig = allConf.getConfiguration(CoreConstant.DATAX_JOB_JOBINFO);
    //初始化PerfTrace
    PerfTrace perfTrace = PerfTrace.getInstance(isJob, instanceId, taskGroupId, priority, traceEnable);
    perfTrace.setJobInfo(jobInfoConfig, perfReportEnable, channelNumber);
    container.start();
}

Example 7 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class JobContainerTest method testErrorLimitIgnoreCheck.

@Test
public void testErrorLimitIgnoreCheck() throws Exception {
    this.configuration.set(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT, -1);
    JobContainer jobContainer = new JobContainer(this.configuration);
    Communication communication = new Communication();
    communication.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 100);
    communication.setLongCounter(CommunicationTool.WRITE_RECEIVED_RECORDS, 100);
    //        LocalTaskGroupCommunicationManager.updateTaskGroupCommunication(0, communication);
    AbstractContainerCommunicator communicator = PowerMockito.mock(AbstractContainerCommunicator.class);
    jobContainer.setContainerCommunicator(communicator);
    PowerMockito.when(communicator.collect()).thenReturn(communication);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("checkLimit");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
}

Example 8 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class JobContainerTest method testStartDryRun.

@Test
public void testStartDryRun() {
    String path = JobContainerTest.class.getClassLoader().getResource(".").getFile();
    this.configuration = ConfigParser.parse(path + File.separator + "dryRunAll.json");
    LoadUtil.bind(this.configuration);
    JobContainer jobContainer = new JobContainer(this.configuration);
    jobContainer.start();
}

Example 9 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class JobContainerTest method testInitNormal.

@Test
public void testInitNormal() throws Exception {
    this.configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID, -2);
    this.configuration.set("runMode", ExecuteMode.STANDALONE.getValue());
    JobContainer jobContainer = new JobContainer(this.configuration);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("init");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    Assert.assertEquals("default job id = 0", 0l, this.configuration.getLong(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID).longValue());
}

Example 10 with JobContainer

use of com.alibaba.datax.core.job.JobContainer in project DataX by alibaba.

the class JobContainerTest method testErrorLimitPercentCheck.

@Test(expected = Exception.class)
public void testErrorLimitPercentCheck() throws Exception {
    //        this.configuration.set(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT, 0.1);
    //        this.configuration.set(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT_RECORD, null);
    this.configuration.remove(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT_RECORD);
    this.configuration.set(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT_PERCENT, 0.1);
    JobContainer jobContainer = new JobContainer(this.configuration);
    Communication communication = new Communication();
    communication.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 100);
    communication.setLongCounter(CommunicationTool.WRITE_RECEIVED_RECORDS, 80);
    communication.setLongCounter(CommunicationTool.WRITE_FAILED_RECORDS, 20);
    //        LocalTaskGroupCommunicationManager.updateTaskGroupCommunication(0, communication);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("checkLimit");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer);
    initMethod.setAccessible(false);
}