Examples with PermitMonitor com.alibaba.otter.shared.arbitrate.impl.setl.monitor.PermitMonitor used on opensource projects

Example 16 with PermitMonitor

use of com.alibaba.otter.shared.arbitrate.impl.setl.monitor.PermitMonitor in project otter by alibaba.

the class ExtractZooKeeperArbitrateEvent method await.

// private TerminArbitrateEvent terminEvent;
/**
     * <pre>
     * 算法:
     * 1. 检查当前的Permit，阻塞等待其授权(解决Channel的pause状态处理)
     * 2. 开始阻塞获取符合条件的processId
     * 3. 检查当前的即时Permit状态 (在阻塞获取processId过程会出现一些error信号,process节点会被删除)
     * 4. 获取Select传递的EventData数据，添加next node信息后直接返回
     * </pre>
     * 
     * @return
     */
public EtlEventData await(Long pipelineId) throws InterruptedException {
    Assert.notNull(pipelineId);
    PermitMonitor permitMonitor = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
    // 阻塞等待授权
    permitMonitor.waitForPermit();
    ExtractStageListener extractStageListener = ArbitrateFactory.getInstance(pipelineId, ExtractStageListener.class);
    // 符合条件的processId
    Long processId = extractStageListener.waitForProcess();
    ChannelStatus status = permitMonitor.getChannelPermit();
    if (status.isStart()) {
        // 即时查询一下当前的状态，状态随时可能会变
        // 根据pipelineId+processId构造对应的path
        String path = StagePathUtils.getSelectStage(pipelineId, processId);
        try {
            byte[] data = zookeeper.readData(path);
            EtlEventData eventData = JsonUtils.unmarshalFromByte(data, EtlEventData.class);
            // 获取下一个处理节点信息
            Node node = LoadBalanceFactory.getNextTransformNode(pipelineId);
            if (node == null) {
                // terminEvent.single(termin);
                throw new ArbitrateException("Extract_single", "no next node");
            } else {
                eventData.setNextNid(node.getId());
                // 只有这一条路返回
                return eventData;
            }
        } catch (ZkNoNodeException e) {
            logger.error("pipeline[{}] processId[{}] is invalid , retry again", pipelineId, processId);
            // /出现节点不存在，说明出现了error情况,递归调用重新获取一次
            return await(pipelineId);
        } catch (ZkException e) {
            throw new ArbitrateException("Extract_await", e.getMessage(), e);
        }
    } else {
        logger.warn("pipelineId[{}] extract ignore processId[{}] by status[{}]", new Object[] { pipelineId, processId, status });
        // 递归调用
        return await(pipelineId);
    }
}

Example 17 with PermitMonitor

use of com.alibaba.otter.shared.arbitrate.impl.setl.monitor.PermitMonitor in project otter by alibaba.

the class LoadZooKeeperArbitrateEvent method await.

// private Map<Long, DistributedLock> locks = new ConcurrentHashMap<Long, DistributedLock>();
/**
     * <pre>
     * 算法:
     * 1. 检查当前的Permit，阻塞等待其授权(解决Channel的pause状态处理)
     * 2. 开始阻塞获取符合条件的processId
     * 3. 检查当前的即时Permit状态 (在阻塞获取processId过程会出现一些error信号,process节点会被删除)
     * 4. 获取Select传递的EventData数据，添加next node信息后直接返回
     * </pre>
     */
public EtlEventData await(Long pipelineId) throws InterruptedException {
    Assert.notNull(pipelineId);
    PermitMonitor permitMonitor = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
    // 阻塞等待授权
    permitMonitor.waitForPermit();
    LoadStageListener loadStageListener = ArbitrateFactory.getInstance(pipelineId, LoadStageListener.class);
    // 符合条件的processId
    Long processId = loadStageListener.waitForProcess();
    // DistributedLock lock = getLock(pipelineId);
    try {
        // 使用锁的理由：
        // 1. 针对双向同步时，其中一个方向出现了异常，需要发起另一端的关闭，此时对方正好在执行某个process的load
        // 2. 单向同步时，如果出现node节点异常，此时正常的节点正在执行某个process的load
        // 为避免因load无法中端引起的数据重复录入，所以针对load阶段添加分布式锁。在有process load过程中不允许进行pipeline关闭操作
        // lock.lock();
        ChannelStatus status = permitMonitor.getChannelPermit();
        if (status.isStart()) {
            // 即时查询一下当前的状态，状态随时可能会变
            // 根据pipelineId+processId构造对应的path
            String path = StagePathUtils.getTransformStage(pipelineId, processId);
            try {
                byte[] data = zookeeper.readData(path);
                // 反序列化并返回
                return JsonUtils.unmarshalFromByte(data, EtlEventData.class);
            } catch (ZkNoNodeException e) {
                logger.error("pipeline[{}] processId[{}] is invalid , retry again", pipelineId, processId);
                // /出现节点不存在，说明出现了error情况,递归调用重新获取一次
                return await(pipelineId);
            } catch (ZkException e) {
                throw e;
            }
        } else {
            logger.warn("pipelineId[{}] load ignore processId[{}] by status[{}]", new Object[] { pipelineId, processId, status });
            // 出现rollback情况，递归调用重新获取一次，当前的processId可丢弃
            return await(pipelineId);
        }
    } catch (InterruptedException e) {
        throw e;
    } catch (Exception e) {
        throw new ArbitrateException(e);
    }
}

Example 18 with PermitMonitor

use of com.alibaba.otter.shared.arbitrate.impl.setl.monitor.PermitMonitor in project otter by alibaba.

the class ArbitrateAllTest method testDemo.

@Test
public void testDemo() {
    // 设置启动标志
    channelEvent.start(channelId);
    // 停顿一下
    sleep();
    PermitMonitor permit = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
    // 优先启动主导线程
    mainStem.submit(pipelineId);
    try {
        // 阻塞等待授权
        permit.waitForPermit();
    } catch (InterruptedException e1) {
        want.fail();
    }
    if (arbitrateMode.isZookeeper()) {
        ArbitrateFactory.getInstance(pipelineId, SelectStageListener.class);
        ArbitrateFactory.getInstance(pipelineId, ExtractStageListener.class);
        ArbitrateFactory.getInstance(pipelineId, TransformStageListener.class);
        ArbitrateFactory.getInstance(pipelineId, LoadStageListener.class);
    }
    if (arbitrateMode.isMemory()) {
        ArbitrateFactory.getInstance(pipelineId, MemoryStageController.class);
    }
    if (arbitrateMode.isRpc()) {
        ArbitrateFactory.getInstance(pipelineId, RpcStageController.class);
        ArbitrateFactory.getInstance(pipelineId, SelectProcessListener.class);
    }
    if (arbitrateMode.isZookeeper() || arbitrateMode.isRpc()) {
        ArbitrateFactory.getInstance(pipelineId, ExtractRandomLoadBanlance.class);
        ArbitrateFactory.getInstance(pipelineId, ExtractRoundRobinLoadBalance.class);
        ArbitrateFactory.getInstance(pipelineId, TransformRandomLoadBanlance.class);
        ArbitrateFactory.getInstance(pipelineId, TransformRoundRobinLoadBalance.class);
    }
    // 启动
    select.submit(pipelineId);
    extract.submit(pipelineId);
    transform.submit(pipelineId);
    load.submit(pipelineId);
    view.submit(pipelineId);
    this.termin.submit(pipelineId);
    try {
        // 运行30s
        Thread.sleep(20 * 1000);
    } catch (InterruptedException e) {
        want.fail();
    }
    // 发送结束事件
    TerminEventData termin = new TerminEventData();
    termin.setPipelineId(pipelineId);
    termin.setType(TerminType.SHUTDOWN);
    arbitrateEventService.terminEvent().single(termin);
    // 等待处理完所有的termin事件
    sleep(5 * 1000L);
    // 关闭
    select.destory(pipelineId);
    extract.destory(pipelineId);
    transform.destory(pipelineId);
    load.destory(pipelineId);
    view.destory(pipelineId);
    this.termin.destory(pipelineId);
    ArbitrateFactory.destory(pipelineId);
}

Example 19 with PermitMonitor

use of com.alibaba.otter.shared.arbitrate.impl.setl.monitor.PermitMonitor in project otter by alibaba.

the class ArbitrateForwardIntegration method testDemo.

@Test
public void testDemo() {
    // 设置启动标志
    channelEvent.start(channelId);
    // 停顿一下
    sleep();
    // 优先启动主导线程
    mainStem.submit(pipelineId);
    PermitMonitor permit = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
    try {
        // 阻塞等待授权
        permit.waitForPermit();
    } catch (InterruptedException e1) {
        want.fail();
    }
    // 启动
    select.submit(pipelineId);
    extract.submit(pipelineId);
    view.submit(pipelineId);
    this.termin.submit(pipelineId);
    // 注意是反方向的
    transform.submit(oppositePipelineId);
    // 注意是反方向的
    load.submit(oppositePipelineId);
    try {
        System.in.read();
    } catch (IOException e) {
        e.printStackTrace();
    }
    // 发送结束事件
    TerminEventData termin = new TerminEventData();
    termin.setPipelineId(pipelineId);
    termin.setType(TerminType.SHUTDOWN);
    arbitrateEventService.terminEvent().single(termin);
    // 等待处理完所有的termin事件
    sleep(5 * 1000L);
    // 关闭
    select.destory(pipelineId);
    extract.destory(pipelineId);
    view.destory(pipelineId);
    this.termin.destory(pipelineId);
    transform.destory(oppositePipelineId);
    load.destory(oppositePipelineId);
    ArbitrateFactory.destory(pipelineId);
}

Example 20 with PermitMonitor

use of com.alibaba.otter.shared.arbitrate.impl.setl.monitor.PermitMonitor in project otter by alibaba.

the class TransformMemoryArbitrateEvent method await.

public EtlEventData await(Long pipelineId) throws InterruptedException {
    Assert.notNull(pipelineId);
    PermitMonitor permitMonitor = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
    // 阻塞等待授权
    permitMonitor.waitForPermit();
    MemoryStageController stageController = ArbitrateFactory.getInstance(pipelineId, MemoryStageController.class);
    // 符合条件的processId
    Long processId = stageController.waitForProcess(StageType.TRANSFORM);
    ChannelStatus status = permitMonitor.getChannelPermit();
    if (status.isStart()) {
        // 即时查询一下当前的状态，状态随时可能会变
        return stageController.getLastData(processId);
    } else {
        logger.warn("pipelineId[{}] transform ignore processId[{}] by status[{}]", new Object[] { pipelineId, processId, status });
        // 递归调用
        return await(pipelineId);
    }
}