use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class SelectMemoryArbitrateEvent method await.
public EtlEventData await(Long pipelineId) throws InterruptedException {
Assert.notNull(pipelineId);
PermitMonitor permitMonitor = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
// 阻塞等待授权
permitMonitor.waitForPermit();
MemoryStageController stageController = ArbitrateFactory.getInstance(pipelineId, MemoryStageController.class);
// 符合条件的processId
Long processId = stageController.waitForProcess(StageType.SELECT);
ChannelStatus status = permitMonitor.getChannelPermit();
if (status.isStart()) {
// 即时查询一下当前的状态,状态随时可能会变
EtlEventData eventData = new EtlEventData();
eventData.setPipelineId(pipelineId);
eventData.setProcessId(processId);
// 返回当前时间
eventData.setStartTime(new Date().getTime());
Long nid = ArbitrateConfigUtils.getCurrentNid();
eventData.setCurrNid(nid);
eventData.setNextNid(nid);
// 只有这一条路返回
return eventData;
} else {
logger.warn("pipelineId[{}] select ignore processId[{}] by status[{}]", new Object[] { pipelineId, processId, status });
//将progress中清理掉,避免阻塞后续调度,因为最小的Id一直处于extract未完成阶段
stageController.clearProgress(processId);
// 递归调用
return await(pipelineId);
}
}
use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class MemoryArbitrateEventTest method test_all.
@Test
public void test_all() {
Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {
@Mock
public int getParallelism(Long pipelineId) {
// 并行度
return 2;
}
});
selectEvent = new SelectMemoryArbitrateEvent();
extractEvent = new ExtractMemoryArbitrateEvent();
transformEvent = new TransformMemoryArbitrateEvent();
loadEvent = new LoadMemoryArbitrateEvent();
terminEvent = (TerminMemoryArbitrateEvent) this.getBeanFactory().getBean("terminMemoryEvent");
loadEvent.setTerminEvent(terminEvent);
final List<Long> initProcessIds = new ArrayList<Long>();
try {
// 获取数据
// select stage
EtlEventData sdata1 = selectEvent.await(pipelineId);
EtlEventData sdata2 = selectEvent.await(pipelineId);
initProcessIds.add(sdata1.getProcessId());
initProcessIds.add(sdata2.getProcessId());
selectEvent.single(sdata1);
selectEvent.single(sdata2);
// extract stage
EtlEventData edata1 = extractEvent.await(pipelineId);
EtlEventData edata2 = extractEvent.await(pipelineId);
extractEvent.single(edata1);
extractEvent.single(edata2);
// transform stage
EtlEventData tdata1 = transformEvent.await(pipelineId);
EtlEventData tdata2 = transformEvent.await(pipelineId);
transformEvent.single(tdata1);
transformEvent.single(tdata2);
// load stage
EtlEventData ldata1 = loadEvent.await(pipelineId);
loadEvent.single(ldata1);
Long p1 = ldata1.getProcessId();
TerminEventData terminData1 = new TerminEventData();
terminData1.setPipelineId(pipelineId);
terminData1.setProcessId(p1);
// 发送ack信号,删除termin节点
terminEvent.ack(terminData1);
EtlEventData ldata2 = loadEvent.await(pipelineId);
want.bool(ldata1.getProcessId() < ldata2.getProcessId()).is(true);
loadEvent.single(ldata2);
Long p2 = ldata2.getProcessId();
TerminEventData terminData2 = new TerminEventData();
terminData2.setPipelineId(pipelineId);
terminData2.setProcessId(p2);
// 发送ack信号,删除termin节点
terminEvent.ack(terminData2);
sleep(2000L);
ArbitrateFactory.destory(pipelineId);
} catch (InterruptedException e) {
want.fail();
} finally {
}
}
use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class TerminMemoryArbitrateEventTest method normalProcess.
private void normalProcess() {
Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {
@Mock
public int getParallelism(Long pipelineId) {
// 并行度
return 2;
}
@Mock
public Long getCurrentNid() {
return nid;
}
});
selectEvent = new SelectMemoryArbitrateEvent();
extractEvent = new ExtractMemoryArbitrateEvent();
transformEvent = new TransformMemoryArbitrateEvent();
loadEvent = new LoadMemoryArbitrateEvent();
terminEvent = (TerminMemoryArbitrateEvent) this.getBeanFactory().getBean("terminMemoryEvent");
loadEvent.setTerminEvent(terminEvent);
final List<Long> initProcessIds = new ArrayList<Long>();
try {
// 获取数据
// select stage
EtlEventData sdata1 = selectEvent.await(pipelineId);
EtlEventData sdata2 = selectEvent.await(pipelineId);
initProcessIds.add(sdata1.getProcessId());
initProcessIds.add(sdata2.getProcessId());
selectEvent.single(sdata1);
selectEvent.single(sdata2);
// extract stage
EtlEventData edata1 = extractEvent.await(pipelineId);
EtlEventData edata2 = extractEvent.await(pipelineId);
extractEvent.single(edata1);
extractEvent.single(edata2);
// transform stage
EtlEventData tdata1 = transformEvent.await(pipelineId);
EtlEventData tdata2 = transformEvent.await(pipelineId);
transformEvent.single(tdata1);
transformEvent.single(tdata2);
// SelectStageListener selectStageListener =
// ArbitrateFactory.getInstance(pipelineId,
// SelectStageListener.class);
// selectStageListener.destory();
// load stage
EtlEventData ldata1 = loadEvent.await(pipelineId);
loadEvent.single(ldata1);
Long p1 = ldata1.getProcessId();
TerminEventData terminData1 = new TerminEventData();
terminData1.setPipelineId(pipelineId);
terminData1.setProcessId(p1);
// 发送ack信号,删除termin节点
terminEvent.ack(terminData1);
} catch (InterruptedException e) {
want.fail();
}
}
use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class RpcArbitrateEventTest method test_all.
@Test
public void test_all() {
Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {
@Mock
public int getParallelism(Long pipelineId) {
// 并行度
return 2;
}
});
RpcStageEventDispatcher rpcStageEventDispatcher = (RpcStageEventDispatcher) this.getBeanFactory().getBean("rpcStageEventDispatcher");
terminEvent = (TerminRpcArbitrateEvent) this.getBeanFactory().getBean("terminRpcEvent");
selectEvent = new SelectRpcArbitrateEvent();
selectEvent.setRpcStageEventDispatcher(rpcStageEventDispatcher);
extractEvent = new ExtractRpcArbitrateEvent();
extractEvent.setRpcStageEventDispatcher(rpcStageEventDispatcher);
transformEvent = new TransformRpcArbitrateEvent();
transformEvent.setRpcStageEventDispatcher(rpcStageEventDispatcher);
loadEvent = new LoadRpcArbitrateEvent();
loadEvent.setRpcStageEventDispatcher(rpcStageEventDispatcher);
loadEvent.setTerminEvent(terminEvent);
final List<Long> initProcessIds = new ArrayList<Long>();
try {
// 获取数据
// select stage
EtlEventData sdata1 = selectEvent.await(pipelineId);
EtlEventData sdata2 = selectEvent.await(pipelineId);
initProcessIds.add(sdata1.getProcessId());
initProcessIds.add(sdata2.getProcessId());
selectEvent.single(sdata1);
selectEvent.single(sdata2);
// extract stage
EtlEventData edata1 = extractEvent.await(pipelineId);
EtlEventData edata2 = extractEvent.await(pipelineId);
extractEvent.single(edata1);
extractEvent.single(edata2);
// transform stage
EtlEventData tdata1 = transformEvent.await(pipelineId);
EtlEventData tdata2 = transformEvent.await(pipelineId);
transformEvent.single(tdata1);
transformEvent.single(tdata2);
// load stage
EtlEventData ldata1 = loadEvent.await(pipelineId);
loadEvent.single(ldata1);
Long p1 = ldata1.getProcessId();
TerminEventData terminData1 = new TerminEventData();
terminData1.setPipelineId(pipelineId);
terminData1.setProcessId(p1);
// 发送ack信号,删除termin节点
terminEvent.ack(terminData1);
EtlEventData ldata2 = loadEvent.await(pipelineId);
want.bool(ldata1.getProcessId() < ldata2.getProcessId()).is(true);
loadEvent.single(ldata2);
Long p2 = ldata2.getProcessId();
TerminEventData terminData2 = new TerminEventData();
terminData2.setPipelineId(pipelineId);
terminData2.setProcessId(p2);
// 发送ack信号,删除termin节点
terminEvent.ack(terminData2);
sleep(2000L);
ArbitrateFactory.destory(pipelineId);
} catch (InterruptedException e) {
want.fail();
} finally {
}
}
use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.
the class SelectArbitrateEventTest method test_select.
@Test
public void test_select() {
Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {
@Mock
public int getParallelism(Long pipelineId) {
//并行度
return 2;
}
});
selectEvent = new SelectZooKeeperArbitrateEvent();
final List<Long> initProcessIds = new ArrayList<Long>();
try {
//获取数据
EtlEventData data1 = selectEvent.await(pipelineId);
EtlEventData data2 = selectEvent.await(pipelineId);
want.bool(data1 != null).is(true);
want.bool(data2 != null).is(true);
initProcessIds.add(data1.getProcessId());
initProcessIds.add(data2.getProcessId());
selectEvent.single(data1);
selectEvent.single(data2);
ArbitrateFactory.destory(pipelineId);
} catch (InterruptedException e) {
want.fail();
} finally {
for (Long processId : initProcessIds) {
destoryStage(processId, ArbitrateConstants.NODE_SELECTED);
destoryProcess(processId);
}
}
}
Aggregations