use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class TerminArbitrateEventTest method normalProcess.
private void normalProcess() {
Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {
@Mock
public int getParallelism(Long pipelineId) {
// 并行度
return 2;
}
@Mock
public Long getCurrentNid() {
return nid;
}
});
selectEvent = new SelectZooKeeperArbitrateEvent();
extractEvent = new ExtractZooKeeperArbitrateEvent();
transformEvent = new TransformZooKeeperArbitrateEvent();
loadEvent = new LoadZooKeeperArbitrateEvent();
terminEvent = (TerminZooKeeperArbitrateEvent) this.getBeanFactory().getBean("terminZooKeeperEvent");
loadEvent.setTerminEvent(terminEvent);
final List<Long> initProcessIds = new ArrayList<Long>();
try {
// 获取数据
// select stage
EtlEventData sdata1 = selectEvent.await(pipelineId);
EtlEventData sdata2 = selectEvent.await(pipelineId);
initProcessIds.add(sdata1.getProcessId());
initProcessIds.add(sdata2.getProcessId());
selectEvent.single(sdata1);
selectEvent.single(sdata2);
// extract stage
EtlEventData edata1 = extractEvent.await(pipelineId);
EtlEventData edata2 = extractEvent.await(pipelineId);
extractEvent.single(edata1);
extractEvent.single(edata2);
// transform stage
EtlEventData tdata1 = transformEvent.await(pipelineId);
EtlEventData tdata2 = transformEvent.await(pipelineId);
transformEvent.single(tdata1);
transformEvent.single(tdata2);
// SelectStageListener selectStageListener =
// ArbitrateFactory.getInstance(pipelineId,
// SelectStageListener.class);
// selectStageListener.destory();
// load stage
EtlEventData ldata1 = loadEvent.await(pipelineId);
loadEvent.single(ldata1);
Long p1 = ldata1.getProcessId();
TerminEventData terminData1 = new TerminEventData();
terminData1.setPipelineId(pipelineId);
terminData1.setProcessId(p1);
// 发送ack信号,删除termin节点
terminEvent.ack(terminData1);
} catch (InterruptedException e) {
want.fail();
}
}
use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class TransformArbitrateEventTest method test_transform.
@Test
public void test_transform() {
Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {
@Mock
public int getParallelism(Long pipelineId) {
// 并行度
return 2;
}
});
selectEvent = new SelectZooKeeperArbitrateEvent();
extractEvent = new ExtractZooKeeperArbitrateEvent();
transformEvent = new TransformZooKeeperArbitrateEvent();
final List<Long> initProcessIds = new ArrayList<Long>();
try {
// 获取数据
// select stage
EtlEventData sdata1 = selectEvent.await(pipelineId);
EtlEventData sdata2 = selectEvent.await(pipelineId);
initProcessIds.add(sdata1.getProcessId());
initProcessIds.add(sdata2.getProcessId());
selectEvent.single(sdata1);
selectEvent.single(sdata2);
// extract stage
EtlEventData edata1 = extractEvent.await(pipelineId);
EtlEventData edata2 = extractEvent.await(pipelineId);
extractEvent.single(edata1);
extractEvent.single(edata2);
// transform stage
EtlEventData tdata1 = transformEvent.await(pipelineId);
EtlEventData tdata2 = transformEvent.await(pipelineId);
transformEvent.single(tdata1);
transformEvent.single(tdata2);
ArbitrateFactory.destory(pipelineId);
} catch (InterruptedException e) {
want.fail();
} finally {
for (Long processId : initProcessIds) {
destoryStage(processId, ArbitrateConstants.NODE_SELECTED);
destoryStage(processId, ArbitrateConstants.NODE_EXTRACTED);
destoryStage(processId, ArbitrateConstants.NODE_TRANSFORMED);
destoryProcess(processId);
}
}
}
use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class BaseStageTest method getData.
protected EtlEventData getData(Long nid) {
EtlEventData data = new EtlEventData();
data.setNextNid(nid);
return data;
}
use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class ExtractTask method run.
public void run() {
MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipelineId));
while (running) {
try {
final EtlEventData etlEventData = arbitrateEventService.extractEvent().await(pipelineId);
Runnable task = new Runnable() {
public void run() {
// 设置profiling信息
boolean profiling = isProfiling();
Long profilingStartTime = null;
if (profiling) {
profilingStartTime = System.currentTimeMillis();
}
MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipelineId));
String currentName = Thread.currentThread().getName();
Thread.currentThread().setName(createTaskName(pipelineId, "ExtractWorker"));
try {
pipeline = configClientService.findPipeline(pipelineId);
List<PipeKey> keys = (List<PipeKey>) etlEventData.getDesc();
long nextNodeId = etlEventData.getNextNid();
DbBatch dbBatch = rowDataPipeDelegate.get(keys);
// 可能拿到为null,因为内存不足或者网络异常,长时间阻塞时,导致从pipe拿数据出现异常,数据可能被上一个节点已经删除
if (dbBatch == null) {
processMissData(pipelineId, "extract miss data with keys:" + keys.toString());
return;
}
// 重新装配一下数据
otterExtractorFactory.extract(dbBatch);
if (dbBatch.getFileBatch() != null && !CollectionUtils.isEmpty(dbBatch.getFileBatch().getFiles()) && pipeline.getParameters().getFileDetect()) {
// 判断一下是否有文件同步,并且需要进行文件对比
// 对比一下中美图片是否有变化
FileBatch fileBatch = fileBatchConflictDetectService.detect(dbBatch.getFileBatch(), nextNodeId);
dbBatch.setFileBatch(fileBatch);
}
List<PipeKey> pipeKeys = rowDataPipeDelegate.put(dbBatch, nextNodeId);
etlEventData.setDesc(pipeKeys);
if (profiling) {
Long profilingEndTime = System.currentTimeMillis();
stageAggregationCollector.push(pipelineId, StageType.EXTRACT, new AggregationItem(profilingStartTime, profilingEndTime));
}
arbitrateEventService.extractEvent().single(etlEventData);
} catch (Throwable e) {
if (!isInterrupt(e)) {
logger.error(String.format("[%d] extractwork executor is error! data:%s", pipelineId, etlEventData), e);
sendRollbackTermin(pipelineId, e);
} else {
logger.info(String.format("[%d] extractwork executor is interrrupt! data:%s", pipelineId, etlEventData), e);
}
} finally {
Thread.currentThread().setName(currentName);
MDC.remove(OtterConstants.splitPipelineLogFileKey);
}
}
};
// 构造pending任务,可在关闭线程时退出任务
SetlFuture extractFuture = new SetlFuture(StageType.EXTRACT, etlEventData.getProcessId(), pendingFuture, task);
executorService.execute(extractFuture);
} catch (Throwable e) {
if (isInterrupt(e)) {
logger.info(String.format("[%s] extractTask is interrupted!", pipelineId), e);
return;
} else {
logger.error(String.format("[%s] extractTask is error!", pipelineId), e);
sendRollbackTermin(pipelineId, e);
}
}
}
}
use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class LoadTask method run.
public void run() {
MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipelineId));
while (running) {
try {
final EtlEventData etlEventData = arbitrateEventService.loadEvent().await(pipelineId);
Runnable task = new Runnable() {
public void run() {
// 设置profiling信息
boolean profiling = isProfiling();
Long profilingStartTime = null;
if (profiling) {
profilingStartTime = System.currentTimeMillis();
}
MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipelineId));
String currentName = Thread.currentThread().getName();
Thread.currentThread().setName(createTaskName(pipelineId, "LoadWorker"));
List<LoadContext> processedContexts = null;
try {
// 后续可判断同步数据是否为rowData
List<PipeKey> keys = (List<PipeKey>) etlEventData.getDesc();
DbBatch dbBatch = rowDataPipeDelegate.get(keys);
// 可能拿到为null,因为内存不足或者网络异常,长时间阻塞时,导致从pipe拿数据出现异常,数据可能被上一个节点已经删除
if (dbBatch == null) {
processMissData(pipelineId, "load miss data with keys:" + keys.toString());
return;
}
// 进行数据load处理
otterLoaderFactory.setStartTime(dbBatch.getRowBatch().getIdentity(), etlEventData.getStartTime());
processedContexts = otterLoaderFactory.load(dbBatch);
if (profiling) {
Long profilingEndTime = System.currentTimeMillis();
stageAggregationCollector.push(pipelineId, StageType.LOAD, new AggregationItem(profilingStartTime, profilingEndTime));
}
// 处理完成后通知single已完成
arbitrateEventService.loadEvent().single(etlEventData);
} catch (Throwable e) {
if (!isInterrupt(e)) {
logger.error(String.format("[%s] loadWork executor is error! data:%s", pipelineId, etlEventData), e);
} else {
logger.info(String.format("[%s] loadWork executor is interrrupt! data:%s", pipelineId, etlEventData), e);
}
if (processedContexts != null) {
// 说明load成功了,但是通知仲裁器失败了,需要记录下记录到store
for (LoadContext context : processedContexts) {
try {
if (context instanceof DbLoadContext) {
dbLoadInterceptor.error((DbLoadContext) context);
}
} catch (Throwable ie) {
logger.error(String.format("[%s] interceptor process error failed!", pipelineId), ie);
}
}
}
if (!isInterrupt(e)) {
sendRollbackTermin(pipelineId, e);
}
} finally {
Thread.currentThread().setName(currentName);
MDC.remove(OtterConstants.splitPipelineLogFileKey);
}
}
};
// 构造pending任务,可在关闭线程时退出任务
SetlFuture extractFuture = new SetlFuture(StageType.LOAD, etlEventData.getProcessId(), pendingFuture, task);
executorService.execute(extractFuture);
} catch (Throwable e) {
if (isInterrupt(e)) {
logger.info(String.format("[%s] loadTask is interrupted!", pipelineId), e);
// 释放锁
return;
} else {
logger.error(String.format("[%s] loadTask is error!", pipelineId), e);
// arbitrateEventService.loadEvent().release(pipelineId); //
// 释放锁
// 先解除lock,后发送rollback信号
sendRollbackTermin(pipelineId, e);
}
}
}
}
Aggregations