use of com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator in project DataX by alibaba.
the class JobContainer method start.
/**
* jobContainer主要负责的工作全部在start()里面,包括init、prepare、split、scheduler、
* post以及destroy和statistics
*/
@Override
public void start() {
LOG.info("DataX jobContainer starts job.");
boolean hasException = false;
boolean isDryRun = false;
try {
this.startTimeStamp = System.currentTimeMillis();
isDryRun = configuration.getBool(CoreConstant.DATAX_JOB_SETTING_DRYRUN, false);
if (isDryRun) {
LOG.info("jobContainer starts to do preCheck ...");
this.preCheck();
} else {
userConf = configuration.clone();
LOG.debug("jobContainer starts to do preHandle ...");
this.preHandle();
LOG.debug("jobContainer starts to do init ...");
this.init();
LOG.info("jobContainer starts to do prepare ...");
this.prepare();
LOG.info("jobContainer starts to do split ...");
this.totalStage = this.split();
LOG.info("jobContainer starts to do schedule ...");
this.schedule();
LOG.debug("jobContainer starts to do post ...");
this.post();
LOG.debug("jobContainer starts to do postHandle ...");
this.postHandle();
LOG.info("DataX jobId [{}] completed successfully.", this.jobId);
this.invokeHooks();
}
} catch (Throwable e) {
LOG.error("Exception when job run", e);
hasException = true;
if (e instanceof OutOfMemoryError) {
this.destroy();
System.gc();
}
if (super.getContainerCommunicator() == null) {
// 由于 containerCollector 是在 scheduler() 中初始化的,所以当在 scheduler() 之前出现异常时,需要在此处对 containerCollector 进行初始化
AbstractContainerCommunicator tempContainerCollector;
// standalone
tempContainerCollector = new StandAloneJobContainerCommunicator(configuration);
super.setContainerCommunicator(tempContainerCollector);
}
Communication communication = super.getContainerCommunicator().collect();
// 汇报前的状态,不需要手动进行设置
// communication.setState(State.FAILED);
communication.setThrowable(e);
communication.setTimestamp(this.endTimeStamp);
Communication tempComm = new Communication();
tempComm.setTimestamp(this.startTransferTimeStamp);
Communication reportCommunication = CommunicationTool.getReportCommunication(communication, tempComm, this.totalStage);
super.getContainerCommunicator().report(reportCommunication);
throw DataXException.asDataXException(FrameworkErrorCode.RUNTIME_ERROR, e);
} finally {
if (!isDryRun) {
this.destroy();
this.endTimeStamp = System.currentTimeMillis();
if (!hasException) {
//最后打印cpu的平均消耗,GC的统计
VMInfo vmInfo = VMInfo.getVmInfo();
if (vmInfo != null) {
vmInfo.getDelta(false);
LOG.info(vmInfo.totalString());
}
LOG.info(PerfTrace.getInstance().summarizeNoException());
this.logStatistics();
}
}
}
}
use of com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator in project DataX by alibaba.
the class JobContainer method initStandaloneScheduler.
private AbstractScheduler initStandaloneScheduler(Configuration configuration) {
AbstractContainerCommunicator containerCommunicator = new StandAloneJobContainerCommunicator(configuration);
super.setContainerCommunicator(containerCommunicator);
return new StandAloneScheduler(containerCommunicator);
}
use of com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator in project DataX by alibaba.
the class JobContainerTest method testErrorLimitIgnoreCheck.
@Test
public void testErrorLimitIgnoreCheck() throws Exception {
this.configuration.set(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT, -1);
JobContainer jobContainer = new JobContainer(this.configuration);
Communication communication = new Communication();
communication.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 100);
communication.setLongCounter(CommunicationTool.WRITE_RECEIVED_RECORDS, 100);
// LocalTaskGroupCommunicationManager.updateTaskGroupCommunication(0, communication);
AbstractContainerCommunicator communicator = PowerMockito.mock(AbstractContainerCommunicator.class);
jobContainer.setContainerCommunicator(communicator);
PowerMockito.when(communicator.collect()).thenReturn(communication);
Method initMethod = jobContainer.getClass().getDeclaredMethod("checkLimit");
initMethod.setAccessible(true);
initMethod.invoke(jobContainer, new Object[] {});
initMethod.setAccessible(false);
}
use of com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator in project DataX by alibaba.
the class TaskGroupContainerTest method testStart.
@Test
public void testStart() throws InterruptedException {
TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configuration);
taskGroupContainer.start();
AbstractContainerCommunicator collector = taskGroupContainer.getContainerCommunicator();
while (true) {
State totalTaskState = collector.collectState();
if (totalTaskState.isRunning()) {
Thread.sleep(1000);
} else {
break;
}
}
Communication totalTaskCommunication = collector.collect();
List<String> messages = totalTaskCommunication.getMessage("bazhen-reader");
Assert.assertTrue(!messages.isEmpty());
messages = totalTaskCommunication.getMessage("bazhen-writer");
Assert.assertTrue(!messages.isEmpty());
messages = totalTaskCommunication.getMessage("bazhen");
Assert.assertNull(messages);
State state = totalTaskCommunication.getState();
Assert.assertTrue("task finished", state.equals(State.SUCCEEDED));
}
Aggregations