Search in sources :

Example 1 with AbstractContainerCommunicator

use of com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator in project DataX by alibaba.

the class JobContainer method start.

/**
     * jobContainer主要负责的工作全部在start()里面,包括init、prepare、split、scheduler、
     * post以及destroy和statistics
     */
@Override
public void start() {
    LOG.info("DataX jobContainer starts job.");
    boolean hasException = false;
    boolean isDryRun = false;
    try {
        this.startTimeStamp = System.currentTimeMillis();
        isDryRun = configuration.getBool(CoreConstant.DATAX_JOB_SETTING_DRYRUN, false);
        if (isDryRun) {
            LOG.info("jobContainer starts to do preCheck ...");
            this.preCheck();
        } else {
            userConf = configuration.clone();
            LOG.debug("jobContainer starts to do preHandle ...");
            this.preHandle();
            LOG.debug("jobContainer starts to do init ...");
            this.init();
            LOG.info("jobContainer starts to do prepare ...");
            this.prepare();
            LOG.info("jobContainer starts to do split ...");
            this.totalStage = this.split();
            LOG.info("jobContainer starts to do schedule ...");
            this.schedule();
            LOG.debug("jobContainer starts to do post ...");
            this.post();
            LOG.debug("jobContainer starts to do postHandle ...");
            this.postHandle();
            LOG.info("DataX jobId [{}] completed successfully.", this.jobId);
            this.invokeHooks();
        }
    } catch (Throwable e) {
        LOG.error("Exception when job run", e);
        hasException = true;
        if (e instanceof OutOfMemoryError) {
            this.destroy();
            System.gc();
        }
        if (super.getContainerCommunicator() == null) {
            // 由于 containerCollector 是在 scheduler() 中初始化的,所以当在 scheduler() 之前出现异常时,需要在此处对 containerCollector 进行初始化
            AbstractContainerCommunicator tempContainerCollector;
            // standalone
            tempContainerCollector = new StandAloneJobContainerCommunicator(configuration);
            super.setContainerCommunicator(tempContainerCollector);
        }
        Communication communication = super.getContainerCommunicator().collect();
        // 汇报前的状态,不需要手动进行设置
        // communication.setState(State.FAILED);
        communication.setThrowable(e);
        communication.setTimestamp(this.endTimeStamp);
        Communication tempComm = new Communication();
        tempComm.setTimestamp(this.startTransferTimeStamp);
        Communication reportCommunication = CommunicationTool.getReportCommunication(communication, tempComm, this.totalStage);
        super.getContainerCommunicator().report(reportCommunication);
        throw DataXException.asDataXException(FrameworkErrorCode.RUNTIME_ERROR, e);
    } finally {
        if (!isDryRun) {
            this.destroy();
            this.endTimeStamp = System.currentTimeMillis();
            if (!hasException) {
                //最后打印cpu的平均消耗,GC的统计
                VMInfo vmInfo = VMInfo.getVmInfo();
                if (vmInfo != null) {
                    vmInfo.getDelta(false);
                    LOG.info(vmInfo.totalString());
                }
                LOG.info(PerfTrace.getInstance().summarizeNoException());
                this.logStatistics();
            }
        }
    }
}
Also used : StandAloneJobContainerCommunicator(com.alibaba.datax.core.statistics.container.communicator.job.StandAloneJobContainerCommunicator) VMInfo(com.alibaba.datax.common.statistics.VMInfo) AbstractContainerCommunicator(com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator) Communication(com.alibaba.datax.core.statistics.communication.Communication)

Example 2 with AbstractContainerCommunicator

use of com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator in project DataX by alibaba.

the class JobContainer method initStandaloneScheduler.

private AbstractScheduler initStandaloneScheduler(Configuration configuration) {
    AbstractContainerCommunicator containerCommunicator = new StandAloneJobContainerCommunicator(configuration);
    super.setContainerCommunicator(containerCommunicator);
    return new StandAloneScheduler(containerCommunicator);
}
Also used : StandAloneJobContainerCommunicator(com.alibaba.datax.core.statistics.container.communicator.job.StandAloneJobContainerCommunicator) StandAloneScheduler(com.alibaba.datax.core.job.scheduler.processinner.StandAloneScheduler) AbstractContainerCommunicator(com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator)

Example 3 with AbstractContainerCommunicator

use of com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator in project DataX by alibaba.

the class JobContainerTest method testErrorLimitIgnoreCheck.

@Test
public void testErrorLimitIgnoreCheck() throws Exception {
    this.configuration.set(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT, -1);
    JobContainer jobContainer = new JobContainer(this.configuration);
    Communication communication = new Communication();
    communication.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 100);
    communication.setLongCounter(CommunicationTool.WRITE_RECEIVED_RECORDS, 100);
    //        LocalTaskGroupCommunicationManager.updateTaskGroupCommunication(0, communication);
    AbstractContainerCommunicator communicator = PowerMockito.mock(AbstractContainerCommunicator.class);
    jobContainer.setContainerCommunicator(communicator);
    PowerMockito.when(communicator.collect()).thenReturn(communication);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("checkLimit");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer, new Object[] {});
    initMethod.setAccessible(false);
}
Also used : JobContainer(com.alibaba.datax.core.job.JobContainer) AbstractContainerCommunicator(com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator) Method(java.lang.reflect.Method) Communication(com.alibaba.datax.core.statistics.communication.Communication) Test(org.junit.Test)

Example 4 with AbstractContainerCommunicator

use of com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator in project DataX by alibaba.

the class TaskGroupContainerTest method testStart.

@Test
public void testStart() throws InterruptedException {
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configuration);
    taskGroupContainer.start();
    AbstractContainerCommunicator collector = taskGroupContainer.getContainerCommunicator();
    while (true) {
        State totalTaskState = collector.collectState();
        if (totalTaskState.isRunning()) {
            Thread.sleep(1000);
        } else {
            break;
        }
    }
    Communication totalTaskCommunication = collector.collect();
    List<String> messages = totalTaskCommunication.getMessage("bazhen-reader");
    Assert.assertTrue(!messages.isEmpty());
    messages = totalTaskCommunication.getMessage("bazhen-writer");
    Assert.assertTrue(!messages.isEmpty());
    messages = totalTaskCommunication.getMessage("bazhen");
    Assert.assertNull(messages);
    State state = totalTaskCommunication.getState();
    Assert.assertTrue("task finished", state.equals(State.SUCCEEDED));
}
Also used : TaskGroupContainer(com.alibaba.datax.core.taskgroup.TaskGroupContainer) State(com.alibaba.datax.dataxservice.face.domain.enums.State) AbstractContainerCommunicator(com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator) Communication(com.alibaba.datax.core.statistics.communication.Communication) Test(org.junit.Test)

Aggregations

AbstractContainerCommunicator (com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator)4 Communication (com.alibaba.datax.core.statistics.communication.Communication)3 StandAloneJobContainerCommunicator (com.alibaba.datax.core.statistics.container.communicator.job.StandAloneJobContainerCommunicator)2 Test (org.junit.Test)2 VMInfo (com.alibaba.datax.common.statistics.VMInfo)1 JobContainer (com.alibaba.datax.core.job.JobContainer)1 StandAloneScheduler (com.alibaba.datax.core.job.scheduler.processinner.StandAloneScheduler)1 TaskGroupContainer (com.alibaba.datax.core.taskgroup.TaskGroupContainer)1 State (com.alibaba.datax.dataxservice.face.domain.enums.State)1 Method (java.lang.reflect.Method)1