Search in sources :

Example 21 with EtlEventData

use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.

the class SelectMemoryArbitrateEvent method await.

public EtlEventData await(Long pipelineId) throws InterruptedException {
    Assert.notNull(pipelineId);
    PermitMonitor permitMonitor = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
    // 阻塞等待授权
    permitMonitor.waitForPermit();
    MemoryStageController stageController = ArbitrateFactory.getInstance(pipelineId, MemoryStageController.class);
    // 符合条件的processId
    Long processId = stageController.waitForProcess(StageType.SELECT);
    ChannelStatus status = permitMonitor.getChannelPermit();
    if (status.isStart()) {
        // 即时查询一下当前的状态,状态随时可能会变
        EtlEventData eventData = new EtlEventData();
        eventData.setPipelineId(pipelineId);
        eventData.setProcessId(processId);
        // 返回当前时间
        eventData.setStartTime(new Date().getTime());
        Long nid = ArbitrateConfigUtils.getCurrentNid();
        eventData.setCurrNid(nid);
        eventData.setNextNid(nid);
        // 只有这一条路返回
        return eventData;
    } else {
        logger.warn("pipelineId[{}] select ignore processId[{}] by status[{}]", new Object[] { pipelineId, processId, status });
        //将progress中清理掉,避免阻塞后续调度,因为最小的Id一直处于extract未完成阶段
        stageController.clearProgress(processId);
        // 递归调用
        return await(pipelineId);
    }
}
Also used : PermitMonitor(com.alibaba.otter.shared.arbitrate.impl.setl.monitor.PermitMonitor) ChannelStatus(com.alibaba.otter.shared.common.model.config.channel.ChannelStatus) Date(java.util.Date) EtlEventData(com.alibaba.otter.shared.arbitrate.model.EtlEventData)

Example 22 with EtlEventData

use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.

the class MemoryArbitrateEventTest method test_all.

@Test
public void test_all() {
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public int getParallelism(Long pipelineId) {
            // 并行度
            return 2;
        }
    });
    selectEvent = new SelectMemoryArbitrateEvent();
    extractEvent = new ExtractMemoryArbitrateEvent();
    transformEvent = new TransformMemoryArbitrateEvent();
    loadEvent = new LoadMemoryArbitrateEvent();
    terminEvent = (TerminMemoryArbitrateEvent) this.getBeanFactory().getBean("terminMemoryEvent");
    loadEvent.setTerminEvent(terminEvent);
    final List<Long> initProcessIds = new ArrayList<Long>();
    try {
        // 获取数据
        // select stage
        EtlEventData sdata1 = selectEvent.await(pipelineId);
        EtlEventData sdata2 = selectEvent.await(pipelineId);
        initProcessIds.add(sdata1.getProcessId());
        initProcessIds.add(sdata2.getProcessId());
        selectEvent.single(sdata1);
        selectEvent.single(sdata2);
        // extract stage
        EtlEventData edata1 = extractEvent.await(pipelineId);
        EtlEventData edata2 = extractEvent.await(pipelineId);
        extractEvent.single(edata1);
        extractEvent.single(edata2);
        // transform stage
        EtlEventData tdata1 = transformEvent.await(pipelineId);
        EtlEventData tdata2 = transformEvent.await(pipelineId);
        transformEvent.single(tdata1);
        transformEvent.single(tdata2);
        // load stage
        EtlEventData ldata1 = loadEvent.await(pipelineId);
        loadEvent.single(ldata1);
        Long p1 = ldata1.getProcessId();
        TerminEventData terminData1 = new TerminEventData();
        terminData1.setPipelineId(pipelineId);
        terminData1.setProcessId(p1);
        // 发送ack信号,删除termin节点
        terminEvent.ack(terminData1);
        EtlEventData ldata2 = loadEvent.await(pipelineId);
        want.bool(ldata1.getProcessId() < ldata2.getProcessId()).is(true);
        loadEvent.single(ldata2);
        Long p2 = ldata2.getProcessId();
        TerminEventData terminData2 = new TerminEventData();
        terminData2.setPipelineId(pipelineId);
        terminData2.setProcessId(p2);
        // 发送ack信号,删除termin节点
        terminEvent.ack(terminData2);
        sleep(2000L);
        ArbitrateFactory.destory(pipelineId);
    } catch (InterruptedException e) {
        want.fail();
    } finally {
    }
}
Also used : TerminEventData(com.alibaba.otter.shared.arbitrate.model.TerminEventData) LoadMemoryArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.memory.LoadMemoryArbitrateEvent) SelectMemoryArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.memory.SelectMemoryArbitrateEvent) ArrayList(java.util.ArrayList) ExtractMemoryArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.memory.ExtractMemoryArbitrateEvent) Mock(mockit.Mock) TransformMemoryArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.memory.TransformMemoryArbitrateEvent) EtlEventData(com.alibaba.otter.shared.arbitrate.model.EtlEventData) Test(org.testng.annotations.Test) BaseArbitrateEventTest(com.alibaba.otter.shared.arbitrate.setl.event.BaseArbitrateEventTest)

Example 23 with EtlEventData

use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.

the class TerminMemoryArbitrateEventTest method normalProcess.

private void normalProcess() {
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public int getParallelism(Long pipelineId) {
            // 并行度
            return 2;
        }

        @Mock
        public Long getCurrentNid() {
            return nid;
        }
    });
    selectEvent = new SelectMemoryArbitrateEvent();
    extractEvent = new ExtractMemoryArbitrateEvent();
    transformEvent = new TransformMemoryArbitrateEvent();
    loadEvent = new LoadMemoryArbitrateEvent();
    terminEvent = (TerminMemoryArbitrateEvent) this.getBeanFactory().getBean("terminMemoryEvent");
    loadEvent.setTerminEvent(terminEvent);
    final List<Long> initProcessIds = new ArrayList<Long>();
    try {
        // 获取数据
        // select stage
        EtlEventData sdata1 = selectEvent.await(pipelineId);
        EtlEventData sdata2 = selectEvent.await(pipelineId);
        initProcessIds.add(sdata1.getProcessId());
        initProcessIds.add(sdata2.getProcessId());
        selectEvent.single(sdata1);
        selectEvent.single(sdata2);
        // extract stage
        EtlEventData edata1 = extractEvent.await(pipelineId);
        EtlEventData edata2 = extractEvent.await(pipelineId);
        extractEvent.single(edata1);
        extractEvent.single(edata2);
        // transform stage
        EtlEventData tdata1 = transformEvent.await(pipelineId);
        EtlEventData tdata2 = transformEvent.await(pipelineId);
        transformEvent.single(tdata1);
        transformEvent.single(tdata2);
        // SelectStageListener selectStageListener =
        // ArbitrateFactory.getInstance(pipelineId,
        // SelectStageListener.class);
        // selectStageListener.destory();
        // load stage
        EtlEventData ldata1 = loadEvent.await(pipelineId);
        loadEvent.single(ldata1);
        Long p1 = ldata1.getProcessId();
        TerminEventData terminData1 = new TerminEventData();
        terminData1.setPipelineId(pipelineId);
        terminData1.setProcessId(p1);
        // 发送ack信号,删除termin节点
        terminEvent.ack(terminData1);
    } catch (InterruptedException e) {
        want.fail();
    }
}
Also used : TerminEventData(com.alibaba.otter.shared.arbitrate.model.TerminEventData) LoadMemoryArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.memory.LoadMemoryArbitrateEvent) SelectMemoryArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.memory.SelectMemoryArbitrateEvent) ArrayList(java.util.ArrayList) ExtractMemoryArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.memory.ExtractMemoryArbitrateEvent) Mock(mockit.Mock) TransformMemoryArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.memory.TransformMemoryArbitrateEvent) EtlEventData(com.alibaba.otter.shared.arbitrate.model.EtlEventData)

Example 24 with EtlEventData

use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.

the class RpcArbitrateEventTest method test_all.

@Test
public void test_all() {
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public int getParallelism(Long pipelineId) {
            // 并行度
            return 2;
        }
    });
    RpcStageEventDispatcher rpcStageEventDispatcher = (RpcStageEventDispatcher) this.getBeanFactory().getBean("rpcStageEventDispatcher");
    terminEvent = (TerminRpcArbitrateEvent) this.getBeanFactory().getBean("terminRpcEvent");
    selectEvent = new SelectRpcArbitrateEvent();
    selectEvent.setRpcStageEventDispatcher(rpcStageEventDispatcher);
    extractEvent = new ExtractRpcArbitrateEvent();
    extractEvent.setRpcStageEventDispatcher(rpcStageEventDispatcher);
    transformEvent = new TransformRpcArbitrateEvent();
    transformEvent.setRpcStageEventDispatcher(rpcStageEventDispatcher);
    loadEvent = new LoadRpcArbitrateEvent();
    loadEvent.setRpcStageEventDispatcher(rpcStageEventDispatcher);
    loadEvent.setTerminEvent(terminEvent);
    final List<Long> initProcessIds = new ArrayList<Long>();
    try {
        // 获取数据
        // select stage
        EtlEventData sdata1 = selectEvent.await(pipelineId);
        EtlEventData sdata2 = selectEvent.await(pipelineId);
        initProcessIds.add(sdata1.getProcessId());
        initProcessIds.add(sdata2.getProcessId());
        selectEvent.single(sdata1);
        selectEvent.single(sdata2);
        // extract stage
        EtlEventData edata1 = extractEvent.await(pipelineId);
        EtlEventData edata2 = extractEvent.await(pipelineId);
        extractEvent.single(edata1);
        extractEvent.single(edata2);
        // transform stage
        EtlEventData tdata1 = transformEvent.await(pipelineId);
        EtlEventData tdata2 = transformEvent.await(pipelineId);
        transformEvent.single(tdata1);
        transformEvent.single(tdata2);
        // load stage
        EtlEventData ldata1 = loadEvent.await(pipelineId);
        loadEvent.single(ldata1);
        Long p1 = ldata1.getProcessId();
        TerminEventData terminData1 = new TerminEventData();
        terminData1.setPipelineId(pipelineId);
        terminData1.setProcessId(p1);
        // 发送ack信号,删除termin节点
        terminEvent.ack(terminData1);
        EtlEventData ldata2 = loadEvent.await(pipelineId);
        want.bool(ldata1.getProcessId() < ldata2.getProcessId()).is(true);
        loadEvent.single(ldata2);
        Long p2 = ldata2.getProcessId();
        TerminEventData terminData2 = new TerminEventData();
        terminData2.setPipelineId(pipelineId);
        terminData2.setProcessId(p2);
        // 发送ack信号,删除termin节点
        terminEvent.ack(terminData2);
        sleep(2000L);
        ArbitrateFactory.destory(pipelineId);
    } catch (InterruptedException e) {
        want.fail();
    } finally {
    }
}
Also used : SelectRpcArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.rpc.SelectRpcArbitrateEvent) ExtractRpcArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.rpc.ExtractRpcArbitrateEvent) TransformRpcArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.rpc.TransformRpcArbitrateEvent) ArrayList(java.util.ArrayList) Mock(mockit.Mock) EtlEventData(com.alibaba.otter.shared.arbitrate.model.EtlEventData) TerminEventData(com.alibaba.otter.shared.arbitrate.model.TerminEventData) LoadRpcArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.rpc.LoadRpcArbitrateEvent) RpcStageEventDispatcher(com.alibaba.otter.shared.arbitrate.impl.setl.rpc.RpcStageEventDispatcher) Test(org.testng.annotations.Test) BaseArbitrateEventTest(com.alibaba.otter.shared.arbitrate.setl.event.BaseArbitrateEventTest)

Example 25 with EtlEventData

use of com.alibaba.otter.shared.arbitrate.model.EtlEventData in project otter by alibaba.

the class SelectArbitrateEventTest method test_select.

@Test
public void test_select() {
    Mockit.setUpMock(ArbitrateConfigUtils.class, new Object() {

        @Mock
        public int getParallelism(Long pipelineId) {
            //并行度
            return 2;
        }
    });
    selectEvent = new SelectZooKeeperArbitrateEvent();
    final List<Long> initProcessIds = new ArrayList<Long>();
    try {
        //获取数据
        EtlEventData data1 = selectEvent.await(pipelineId);
        EtlEventData data2 = selectEvent.await(pipelineId);
        want.bool(data1 != null).is(true);
        want.bool(data2 != null).is(true);
        initProcessIds.add(data1.getProcessId());
        initProcessIds.add(data2.getProcessId());
        selectEvent.single(data1);
        selectEvent.single(data2);
        ArbitrateFactory.destory(pipelineId);
    } catch (InterruptedException e) {
        want.fail();
    } finally {
        for (Long processId : initProcessIds) {
            destoryStage(processId, ArbitrateConstants.NODE_SELECTED);
            destoryProcess(processId);
        }
    }
}
Also used : SelectZooKeeperArbitrateEvent(com.alibaba.otter.shared.arbitrate.impl.setl.zookeeper.SelectZooKeeperArbitrateEvent) ArrayList(java.util.ArrayList) Mock(mockit.Mock) EtlEventData(com.alibaba.otter.shared.arbitrate.model.EtlEventData) BaseArbitrateEventTest(com.alibaba.otter.shared.arbitrate.setl.event.BaseArbitrateEventTest) Test(org.testng.annotations.Test)

Aggregations

EtlEventData (com.alibaba.otter.shared.arbitrate.model.EtlEventData)25 ArrayList (java.util.ArrayList)10 Mock (mockit.Mock)8 PermitMonitor (com.alibaba.otter.shared.arbitrate.impl.setl.monitor.PermitMonitor)7 TerminEventData (com.alibaba.otter.shared.arbitrate.model.TerminEventData)7 ChannelStatus (com.alibaba.otter.shared.common.model.config.channel.ChannelStatus)7 ZkException (org.I0Itec.zkclient.exception.ZkException)7 ZkNoNodeException (org.I0Itec.zkclient.exception.ZkNoNodeException)7 ArbitrateException (com.alibaba.otter.shared.arbitrate.exception.ArbitrateException)6 BaseArbitrateEventTest (com.alibaba.otter.shared.arbitrate.setl.event.BaseArbitrateEventTest)6 Test (org.testng.annotations.Test)6 SelectZooKeeperArbitrateEvent (com.alibaba.otter.shared.arbitrate.impl.setl.zookeeper.SelectZooKeeperArbitrateEvent)5 AggregationItem (com.alibaba.otter.node.etl.common.jmx.StageAggregation.AggregationItem)4 PipeKey (com.alibaba.otter.node.etl.common.pipe.PipeKey)4 ExtractZooKeeperArbitrateEvent (com.alibaba.otter.shared.arbitrate.impl.setl.zookeeper.ExtractZooKeeperArbitrateEvent)4 Node (com.alibaba.otter.shared.common.model.config.node.Node)4 DbBatch (com.alibaba.otter.shared.etl.model.DbBatch)4 SetlFuture (com.alibaba.otter.node.etl.extract.SetlFuture)3 TransformZooKeeperArbitrateEvent (com.alibaba.otter.shared.arbitrate.impl.setl.zookeeper.TransformZooKeeperArbitrateEvent)3 Date (java.util.Date)3