Examples with LoadStageListener com.alibaba.otter.shared.arbitrate.impl.setl.zookeeper.monitor.LoadStageListener used on opensource projects

Example 1 with LoadStageListener

use of com.alibaba.otter.shared.arbitrate.impl.setl.zookeeper.monitor.LoadStageListener in project otter by alibaba.

the class LoadStageListenerTest method testProcess_init.

@Test
public void testProcess_init() {
    final List<Long> initProcessIds = new ArrayList<Long>();
    final Map<Long, List<String>> stages = new HashMap<Long, List<String>>();
    try {
        // 准备数据
        Long p1 = initProcess();
        initStage(p1, ArbitrateConstants.NODE_SELECTED);
        initStage(p1, ArbitrateConstants.NODE_EXTRACTED);
        initStage(p1, ArbitrateConstants.NODE_TRANSFORMED, getData(nid));
        Long p2 = initProcess();
        initStage(p2, ArbitrateConstants.NODE_SELECTED);
        initStage(p2, ArbitrateConstants.NODE_EXTRACTED);
        initStage(p2, ArbitrateConstants.NODE_TRANSFORMED, getData(nid + 1));
        // 准备清理数据
        initProcessIds.add(p1);
        initProcessIds.add(p2);
        List<String> p1Stages = Arrays.asList(ArbitrateConstants.NODE_SELECTED, ArbitrateConstants.NODE_EXTRACTED, ArbitrateConstants.NODE_TRANSFORMED);
        stages.put(p1, p1Stages);
        List<String> p2Stages = Arrays.asList(ArbitrateConstants.NODE_SELECTED, ArbitrateConstants.NODE_EXTRACTED, ArbitrateConstants.NODE_TRANSFORMED);
        stages.put(p2, p2Stages);
        // 进行验证
        LoadStageListener load = new LoadStageListener(pipelineId);
        Long processId = load.waitForProcess();
        want.number(processId).isEqualTo(p1);
        // 验证下process信息
        StageMonitor monitor = ArbitrateFactory.getInstance(pipelineId, StageMonitor.class);
        List<Long> processIds = monitor.getCurrentProcessIds();
        // 获取下stage信息
        List<String> currentP1Stages = monitor.getCurrentStages(p1);
        List<String> currentP2Stages = monitor.getCurrentStages(p2);
        want.collection(processIds).isEqualTo(initProcessIds);
        want.collection(currentP1Stages).isEqualTo(stages.get(p1));
        want.collection(currentP2Stages).isEqualTo(stages.get(p2));
        load.destory();
        ArbitrateFactory.destory(pipelineId);
    } catch (InterruptedException e) {
        want.fail();
    } finally {
        for (Long processId : initProcessIds) {
            List<String> ss = stages.get(processId);
            for (String stage : ss) {
                destoryStage(processId, stage);
            }
            destoryProcess(processId);
        }
    }
}

Example 2 with LoadStageListener

use of com.alibaba.otter.shared.arbitrate.impl.setl.zookeeper.monitor.LoadStageListener in project otter by alibaba.

the class LoadZooKeeperArbitrateEvent method await.

// private Map<Long, DistributedLock> locks = new ConcurrentHashMap<Long,
// DistributedLock>();
/**
 * <pre>
 * 算法:
 * 1. 检查当前的Permit，阻塞等待其授权(解决Channel的pause状态处理)
 * 2. 开始阻塞获取符合条件的processId
 * 3. 检查当前的即时Permit状态 (在阻塞获取processId过程会出现一些error信号,process节点会被删除)
 * 4. 获取Select传递的EventData数据，添加next node信息后直接返回
 * </pre>
 */
public EtlEventData await(Long pipelineId) throws InterruptedException {
    Assert.notNull(pipelineId);
    PermitMonitor permitMonitor = ArbitrateFactory.getInstance(pipelineId, PermitMonitor.class);
    // 阻塞等待授权
    permitMonitor.waitForPermit();
    LoadStageListener loadStageListener = ArbitrateFactory.getInstance(pipelineId, LoadStageListener.class);
    // 符合条件的processId
    Long processId = loadStageListener.waitForProcess();
    // DistributedLock lock = getLock(pipelineId);
    try {
        // 使用锁的理由：
        // 1. 针对双向同步时，其中一个方向出现了异常，需要发起另一端的关闭，此时对方正好在执行某个process的load
        // 2. 单向同步时，如果出现node节点异常，此时正常的节点正在执行某个process的load
        // 为避免因load无法中端引起的数据重复录入，所以针对load阶段添加分布式锁。在有process
        // load过程中不允许进行pipeline关闭操作
        // lock.lock();
        ChannelStatus status = permitMonitor.getChannelPermit();
        if (status.isStart()) {
            // 即时查询一下当前的状态，状态随时可能会变
            // 根据pipelineId+processId构造对应的path
            String path = StagePathUtils.getTransformStage(pipelineId, processId);
            try {
                byte[] data = zookeeper.readData(path);
                // 反序列化并返回
                return JsonUtils.unmarshalFromByte(data, EtlEventData.class);
            } catch (ZkNoNodeException e) {
                logger.error("pipeline[{}] processId[{}] is invalid , retry again", pipelineId, processId);
                // /出现节点不存在，说明出现了error情况,递归调用重新获取一次
                return await(pipelineId);
            } catch (ZkException e) {
                throw e;
            }
        } else {
            logger.warn("pipelineId[{}] load ignore processId[{}] by status[{}]", new Object[] { pipelineId, processId, status });
            // try {
            // lock.unlock();// 出现任何异常解除lock
            // } catch (KeeperException e) {
            // // ignore
            // }
            // 释放下processId，因为load是等待processId最小值完成Tranform才继续，如果这里不释放，会一直卡死等待
            String path = StagePathUtils.getProcess(pipelineId, processId);
            zookeeper.delete(path);
            // 出现rollback情况，递归调用重新获取一次，当前的processId可丢弃
            return await(pipelineId);
        }
    } catch (InterruptedException e) {
        throw e;
    } catch (Exception e) {
        throw new ArbitrateException(e);
    }
}

Example 3 with LoadStageListener

use of com.alibaba.otter.shared.arbitrate.impl.setl.zookeeper.monitor.LoadStageListener in project otter by alibaba.

the class LoadStageListenerTest method testProcess_dymanic.

@Test
public void testProcess_dymanic() {
    final List<Long> initProcessIds = new ArrayList<Long>();
    final Map<Long, List<String>> stages = new HashMap<Long, List<String>>();
    try {
        // 准备数据
        Long p1 = initProcess();
        initStage(p1, ArbitrateConstants.NODE_SELECTED);
        initStage(p1, ArbitrateConstants.NODE_EXTRACTED);
        initStage(p1, ArbitrateConstants.NODE_TRANSFORMED, getData(nid + 1));
        Long p2 = initProcess();
        initStage(p2, ArbitrateConstants.NODE_SELECTED);
        initStage(p2, ArbitrateConstants.NODE_EXTRACTED);
        initStage(p2, ArbitrateConstants.NODE_TRANSFORMED, getData(nid));
        Long p3 = initProcess();
        initStage(p3, ArbitrateConstants.NODE_SELECTED);
        initStage(p3, ArbitrateConstants.NODE_EXTRACTED);
        initStage(p3, ArbitrateConstants.NODE_TRANSFORMED, getData(nid));
        // 初始化
        LoadStageListener load = new LoadStageListener(pipelineId);
        // 开始变化
        // initStage(p1, ArbitrateConstants.NODE_LOADED);
        destoryStage(p1, ArbitrateConstants.NODE_SELECTED);
        destoryStage(p1, ArbitrateConstants.NODE_EXTRACTED);
        destoryStage(p1, ArbitrateConstants.NODE_TRANSFORMED);
        // destoryStage(p1, ArbitrateConstants.NODE_LOADED);
        destoryProcess(p1);
        // 准备清理数据
        initProcessIds.add(p2);
        initProcessIds.add(p3);
        List<String> p1Stages = Lists.newArrayList();
        stages.put(p1, p1Stages);
        List<String> p2Stages = Arrays.asList(ArbitrateConstants.NODE_SELECTED, ArbitrateConstants.NODE_EXTRACTED, ArbitrateConstants.NODE_TRANSFORMED);
        stages.put(p2, p2Stages);
        List<String> p3Stages = Arrays.asList(ArbitrateConstants.NODE_SELECTED, ArbitrateConstants.NODE_EXTRACTED, ArbitrateConstants.NODE_TRANSFORMED);
        stages.put(p3, p3Stages);
        sleep();
        // 进行验证
        Long processId = load.waitForProcess();
        want.number(processId).isEqualTo(p2);
        // 验证下process信息
        StageMonitor monitor = ArbitrateFactory.getInstance(pipelineId, StageMonitor.class);
        List<Long> processIds = monitor.getCurrentProcessIds();
        // 获取下stage信息
        List<String> currentP1Stages = monitor.getCurrentStages(p1);
        List<String> currentP2Stages = monitor.getCurrentStages(p2);
        List<String> currentP3Stages = monitor.getCurrentStages(p3);
        want.collection(processIds).isEqualTo(initProcessIds);
        want.collection(currentP1Stages).isEqualTo(stages.get(p1));
        want.collection(currentP2Stages).isEqualTo(stages.get(p2));
        want.collection(currentP3Stages).isEqualTo(stages.get(p3));
        // 继续变化
        // initStage(p2, ArbitrateConstants.NODE_LOADED);
        destoryStage(p2, ArbitrateConstants.NODE_SELECTED);
        destoryStage(p2, ArbitrateConstants.NODE_EXTRACTED);
        destoryStage(p2, ArbitrateConstants.NODE_TRANSFORMED);
        // destoryStage(p2, ArbitrateConstants.NODE_LOADED);
        destoryProcess(p2);
        // 准备清理数据
        initProcessIds.remove(p2);
        p2Stages = Lists.newArrayList();
        stages.put(p2, p2Stages);
        sleep();
        // 进行验证
        processId = load.waitForProcess();
        want.number(processId).isEqualTo(p3);
        // 验证下process信息
        processIds = monitor.getCurrentProcessIds();
        // 获取下stage信息
        currentP1Stages = monitor.getCurrentStages(p1);
        currentP2Stages = monitor.getCurrentStages(p2);
        currentP3Stages = monitor.getCurrentStages(p3);
        want.collection(processIds).isEqualTo(initProcessIds);
        want.collection(currentP1Stages).isEqualTo(stages.get(p1));
        want.collection(currentP2Stages).isEqualTo(stages.get(p2));
        want.collection(currentP3Stages).isEqualTo(stages.get(p3));
        load.destory();
        ArbitrateFactory.destory(pipelineId);
    } catch (InterruptedException e) {
        want.fail();
    } finally {
        for (Long processId : initProcessIds) {
            List<String> ss = stages.get(processId);
            for (String stage : ss) {
                destoryStage(processId, stage);
            }
            destoryProcess(processId);
        }
    }
}