Examples with StageComparator com.alibaba.otter.shared.arbitrate.impl.setl.helper.StageComparator used on opensource projects

Example 1 with StageComparator

use of com.alibaba.otter.shared.arbitrate.impl.setl.helper.StageComparator in project otter by alibaba.

the class ArbitrateViewServiceImpl method listProcesses.

public List<ProcessStat> listProcesses(Long channelId, Long pipelineId) {
    List<ProcessStat> processStats = new ArrayList<ProcessStat>();
    String processRoot = ManagePathUtils.getProcessRoot(channelId, pipelineId);
    IZkConnection connection = zookeeper.getConnection();
    // zkclient会将获取stat信息和正常的操作分开，使用原生的zk进行优化
    ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();
    // 获取所有的process列表
    List<String> processNodes = zookeeper.getChildren(processRoot);
    List<Long> processIds = new ArrayList<Long>();
    for (String processNode : processNodes) {
        processIds.add(ManagePathUtils.getProcessId(processNode));
    }
    Collections.sort(processIds);
    for (int i = 0; i < processIds.size(); i++) {
        Long processId = processIds.get(i);
        // 当前的process可能会有变化
        ProcessStat processStat = new ProcessStat();
        processStat.setPipelineId(pipelineId);
        processStat.setProcessId(processId);
        List<StageStat> stageStats = new ArrayList<StageStat>();
        processStat.setStageStats(stageStats);
        try {
            String processPath = ManagePathUtils.getProcess(channelId, pipelineId, processId);
            Stat zkProcessStat = new Stat();
            List<String> stages = orginZk.getChildren(processPath, false, zkProcessStat);
            Collections.sort(stages, new StageComparator());
            StageStat prev = null;
            for (String stage : stages) {
                // 循环每个process下的stage
                String stagePath = processPath + "/" + stage;
                Stat zkStat = new Stat();
                StageStat stageStat = new StageStat();
                stageStat.setPipelineId(pipelineId);
                stageStat.setProcessId(processId);
                byte[] bytes = orginZk.getData(stagePath, false, zkStat);
                if (bytes != null && bytes.length > 0) {
                    // 特殊处理zookeeper里的data信息，manager没有对应node中PipeKey的对象，所以导致反序列化会失败，需要特殊处理，删除'@'符号
                    String json = StringUtils.remove(new String(bytes, "UTF-8"), '@');
                    EtlEventData data = JsonUtils.unmarshalFromString(json, EtlEventData.class);
                    stageStat.setNumber(data.getNumber());
                    stageStat.setSize(data.getSize());
                    Map exts = new HashMap();
                    if (!CollectionUtils.isEmpty(data.getExts())) {
                        exts.putAll(data.getExts());
                    }
                    exts.put("currNid", data.getCurrNid());
                    exts.put("nextNid", data.getNextNid());
                    exts.put("desc", data.getDesc());
                    stageStat.setExts(exts);
                }
                if (prev != null) {
                    // 对应的start时间为上一个节点的结束时间
                    stageStat.setStartTime(prev.getEndTime());
                } else {
                    // process的最后修改时间,select
                    stageStat.setStartTime(zkProcessStat.getMtime());
                // await成功后会设置USED标志位
                }
                stageStat.setEndTime(zkStat.getMtime());
                if (ArbitrateConstants.NODE_SELECTED.equals(stage)) {
                    stageStat.setStage(StageType.SELECT);
                } else if (ArbitrateConstants.NODE_EXTRACTED.equals(stage)) {
                    stageStat.setStage(StageType.EXTRACT);
                } else if (ArbitrateConstants.NODE_TRANSFORMED.equals(stage)) {
                    stageStat.setStage(StageType.TRANSFORM);
                // } else if
                // (ArbitrateConstants.NODE_LOADED.equals(stage)) {
                // stageStat.setStage(StageType.LOAD);
                }
                prev = stageStat;
                stageStats.add(stageStat);
            }
            // 添加一个当前正在处理的
            StageStat currentStageStat = new StageStat();
            currentStageStat.setPipelineId(pipelineId);
            currentStageStat.setProcessId(processId);
            if (prev == null) {
                byte[] bytes = orginZk.getData(processPath, false, zkProcessStat);
                if (bytes == null || bytes.length == 0) {
                    // 直接认为未使用，忽略之
                    continue;
                }
                ProcessNodeEventData nodeData = JsonUtils.unmarshalFromByte(bytes, ProcessNodeEventData.class);
                if (nodeData.getStatus().isUnUsed()) {
                    // 跳过该process
                    continue;
                } else {
                    // select操作
                    currentStageStat.setStage(StageType.SELECT);
                    currentStageStat.setStartTime(zkProcessStat.getMtime());
                }
            } else {
                // 判断上一个节点，确定当前的stage
                StageType stage = prev.getStage();
                if (stage.isSelect()) {
                    currentStageStat.setStage(StageType.EXTRACT);
                } else if (stage.isExtract()) {
                    currentStageStat.setStage(StageType.TRANSFORM);
                } else if (stage.isTransform()) {
                    currentStageStat.setStage(StageType.LOAD);
                } else if (stage.isLoad()) {
                    // 已经是最后一个节点了
                    continue;
                }
                // 开始时间为上一个节点的结束时间
                currentStageStat.setStartTime(prev.getEndTime());
            }
            if (currentStageStat.getStage().isLoad()) {
                // load必须为第一个process节点
                if (i == 0) {
                    stageStats.add(currentStageStat);
                }
            } else {
                // 其他情况都添加
                stageStats.add(currentStageStat);
            }
        } catch (NoNodeException e) {
        // ignore
        } catch (KeeperException e) {
            throw new ArbitrateException(e);
        } catch (InterruptedException e) {
        // ignore
        } catch (UnsupportedEncodingException e) {
        // ignore
        }
        processStats.add(processStat);
    }
    return processStats;
}

Example 2 with StageComparator

use of com.alibaba.otter.shared.arbitrate.impl.setl.helper.StageComparator in project otter by alibaba.

the class StageMonitor method initProcessStage.

/**
     * 处理指定的processId变化，返回结果true代表需要继续监听,false代表不需要监视
     */
private boolean initProcessStage(Long processId, List<String> currentStages) {
    Collections.sort(currentStages, new StageComparator());
    if (logger.isDebugEnabled()) {
        logger.debug("pipeline[{}] processId[{}] with stage{}", new Object[] { getPipelineId(), processId, currentStages });
    }
    // 更新下stage数据
    this.currentStages.put(processId, currentStages);
    // 通知stage变化
    stageChanged(processId, currentStages);
    // 2.1 判断是否存在了loaded节点，此节点为最后一个节点
    if (currentStages.contains(ArbitrateConstants.NODE_TRANSFORMED)) {
        // 不需要监听了
        return true;
    } else {
        // 其他的状态需要监听
        return false;
    }
}

Example 3 with StageComparator

use of com.alibaba.otter.shared.arbitrate.impl.setl.helper.StageComparator in project otter by alibaba.

the class StageMonitor method syncStage.

// /**
// * 监听process的新增/删除变化，触发process列表的更新
// */
// private void syncStage() {
// // 1. 根据pipelineId构造对应的path
// String path = null;
// try {
// path = StagePathUtils.getProcessRoot(getPipelineId());
// // 2. 监听当前的process列表的变化
// List<String> currentProcesses = zookeeper.getChildren(path, new AsyncWatcher() {
//
// public void asyncProcess(WatchedEvent event) {
// MDC.put(ArbitrateConstants.splitPipelineLogFileKey, String.valueOf(getPipelineId()));
// if (isStop()) {
// return;
// }
//
// // 出现session expired/connection losscase下，会触发所有的watcher响应，同时老的watcher会继续保留，所以会导致出现多次watcher响应
// boolean dataChanged = event.getType() == EventType.NodeDataChanged
// || event.getType() == EventType.NodeDeleted
// || event.getType() == EventType.NodeCreated
// || event.getType() == EventType.NodeChildrenChanged;
// if (dataChanged) {
// syncStage(); // 继续监听
// // initStage(); // 重新更新
// }
// }
// });
//
// // 3. 循环处理每个process
// List<Long> processIds = new ArrayList<Long>();
// for (String process : currentProcesses) {
// processIds.add(StagePathUtils.getProcessId(process));
// }
//
// Collections.sort(processIds); // 排序一下
// // 判断一下当前processIds和当前内存中的记录是否有差异，如果有差异立马触发一下
// if (!currentProcessIds.equals(processIds)) {
// initStage(); // 立马触发一下
// }
// } catch (KeeperException e) {
// syncStage(); // 继续监听
// } catch (InterruptedException e) {
// // ignore
// }
// }
/**
     * 监听指定的processId节点的变化
     */
private void syncStage(final Long processId) {
    // 1. 根据pipelineId + processId构造对应的path
    String path = null;
    try {
        path = StagePathUtils.getProcess(getPipelineId(), processId);
        // 2. 监听当前的process列表的变化
        IZkConnection connection = zookeeper.getConnection();
        // zkclient包装的是一个持久化的zk，分布式lock只需要一次性的watcher，需要调用原始的zk链接进行操作
        ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();
        List<String> currentStages = orginZk.getChildren(path, new AsyncWatcher() {

            public void asyncProcess(WatchedEvent event) {
                MDC.put(ArbitrateConstants.splitPipelineLogFileKey, String.valueOf(getPipelineId()));
                if (isStop()) {
                    return;
                }
                if (event.getType() == EventType.NodeDeleted) {
                    // 触发下节点删除
                    processTermined(processId);
                    return;
                }
                // 出现session expired/connection losscase下，会触发所有的watcher响应，同时老的watcher会继续保留，所以会导致出现多次watcher响应
                boolean dataChanged = event.getType() == EventType.NodeDataChanged || event.getType() == EventType.NodeDeleted || event.getType() == EventType.NodeCreated || event.getType() == EventType.NodeChildrenChanged;
                if (dataChanged) {
                    // boolean reply = initStage(processId);
                    // if (reply == false) {// 出现过load后就不需要再监听变化，剩下的就是节点的删除操作
                    syncStage(processId);
                // }
                }
            }
        });
        Collections.sort(currentStages, new StageComparator());
        List<String> lastStages = this.currentStages.get(processId);
        if (lastStages == null || !lastStages.equals(currentStages)) {
            // 存在差异，立马触发一下
            initProcessStage(processId);
        }
    } catch (NoNodeException e) {
        // 触发下节点删除
        processTermined(processId);
    } catch (KeeperException e) {
        syncStage(processId);
    } catch (InterruptedException e) {
    // ignore
    }
}

Example 4 with StageComparator

use of com.alibaba.otter.shared.arbitrate.impl.setl.helper.StageComparator in project otter by alibaba.

the class NormalTerminProcess method doProcess.

private boolean doProcess(TerminEventData data, boolean retry) {
    Long pipelineId = data.getPipelineId();
    Long processId = data.getProcessId();
    List<String> currentStages = null;
    try {
        currentStages = zookeeper.getChildren(StagePathUtils.getProcess(pipelineId, processId));
        Collections.sort(currentStages, new StageComparator());
    } catch (ZkNoNodeException e) {
        // ignore，说明节点已经被删除了
        return false;
    } catch (ZkException e) {
        throw new ArbitrateException("Termin_process", e);
    }
    // s节点
    if (currentStages == null || currentStages.contains(ArbitrateConstants.NODE_SELECTED)) {
        try {
            boolean successed = zookeeper.delete(StagePathUtils.getSelectStage(pipelineId, processId));
            if (!successed) {
                processDeleteFailed();
            }
        } catch (ZkException e) {
            throw new ArbitrateException("Termin_process", e);
        }
    }
    // e节点
    if (currentStages == null || currentStages.contains(ArbitrateConstants.NODE_EXTRACTED)) {
        try {
            boolean successed = zookeeper.delete(StagePathUtils.getExtractStage(pipelineId, processId));
            if (!successed) {
                processDeleteFailed();
            }
        } catch (ZkException e) {
            throw new ArbitrateException("Termin_process", e);
        }
    }
    // t节点
    if (currentStages == null || currentStages.contains(ArbitrateConstants.NODE_TRANSFORMED)) {
        try {
            boolean successed = zookeeper.delete(StagePathUtils.getTransformStage(pipelineId, processId));
            if (!successed) {
                processDeleteFailed();
            }
        } catch (ZkException e) {
            throw new ArbitrateException("Termin_process", e);
        }
    }
    // 不过会有遗漏判断，比如并发时都是一个线程全删除成功
    return processDelete(data, CollectionUtils.isEmpty(currentStages), retry);
}