Examples with StageType com.alibaba.otter.shared.common.model.config.enums.StageType used on opensource projects

Example 1 with StageType

use of com.alibaba.otter.shared.common.model.config.enums.StageType in project otter by alibaba.

the class ArbitrateViewServiceImpl method listProcesses.

public List<ProcessStat> listProcesses(Long channelId, Long pipelineId) {
    List<ProcessStat> processStats = new ArrayList<ProcessStat>();
    String processRoot = ManagePathUtils.getProcessRoot(channelId, pipelineId);
    IZkConnection connection = zookeeper.getConnection();
    // zkclient会将获取stat信息和正常的操作分开，使用原生的zk进行优化
    ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();
    // 获取所有的process列表
    List<String> processNodes = zookeeper.getChildren(processRoot);
    List<Long> processIds = new ArrayList<Long>();
    for (String processNode : processNodes) {
        processIds.add(ManagePathUtils.getProcessId(processNode));
    }
    Collections.sort(processIds);
    for (int i = 0; i < processIds.size(); i++) {
        Long processId = processIds.get(i);
        // 当前的process可能会有变化
        ProcessStat processStat = new ProcessStat();
        processStat.setPipelineId(pipelineId);
        processStat.setProcessId(processId);
        List<StageStat> stageStats = new ArrayList<StageStat>();
        processStat.setStageStats(stageStats);
        try {
            String processPath = ManagePathUtils.getProcess(channelId, pipelineId, processId);
            Stat zkProcessStat = new Stat();
            List<String> stages = orginZk.getChildren(processPath, false, zkProcessStat);
            Collections.sort(stages, new StageComparator());
            StageStat prev = null;
            for (String stage : stages) {
                // 循环每个process下的stage
                String stagePath = processPath + "/" + stage;
                Stat zkStat = new Stat();
                StageStat stageStat = new StageStat();
                stageStat.setPipelineId(pipelineId);
                stageStat.setProcessId(processId);
                byte[] bytes = orginZk.getData(stagePath, false, zkStat);
                if (bytes != null && bytes.length > 0) {
                    // 特殊处理zookeeper里的data信息，manager没有对应node中PipeKey的对象，所以导致反序列化会失败，需要特殊处理，删除'@'符号
                    String json = StringUtils.remove(new String(bytes, "UTF-8"), '@');
                    EtlEventData data = JsonUtils.unmarshalFromString(json, EtlEventData.class);
                    stageStat.setNumber(data.getNumber());
                    stageStat.setSize(data.getSize());
                    Map exts = new HashMap();
                    if (!CollectionUtils.isEmpty(data.getExts())) {
                        exts.putAll(data.getExts());
                    }
                    exts.put("currNid", data.getCurrNid());
                    exts.put("nextNid", data.getNextNid());
                    exts.put("desc", data.getDesc());
                    stageStat.setExts(exts);
                }
                if (prev != null) {
                    // 对应的start时间为上一个节点的结束时间
                    stageStat.setStartTime(prev.getEndTime());
                } else {
                    // process的最后修改时间,select
                    stageStat.setStartTime(zkProcessStat.getMtime());
                // await成功后会设置USED标志位
                }
                stageStat.setEndTime(zkStat.getMtime());
                if (ArbitrateConstants.NODE_SELECTED.equals(stage)) {
                    stageStat.setStage(StageType.SELECT);
                } else if (ArbitrateConstants.NODE_EXTRACTED.equals(stage)) {
                    stageStat.setStage(StageType.EXTRACT);
                } else if (ArbitrateConstants.NODE_TRANSFORMED.equals(stage)) {
                    stageStat.setStage(StageType.TRANSFORM);
                // } else if
                // (ArbitrateConstants.NODE_LOADED.equals(stage)) {
                // stageStat.setStage(StageType.LOAD);
                }
                prev = stageStat;
                stageStats.add(stageStat);
            }
            // 添加一个当前正在处理的
            StageStat currentStageStat = new StageStat();
            currentStageStat.setPipelineId(pipelineId);
            currentStageStat.setProcessId(processId);
            if (prev == null) {
                byte[] bytes = orginZk.getData(processPath, false, zkProcessStat);
                if (bytes == null || bytes.length == 0) {
                    // 直接认为未使用，忽略之
                    continue;
                }
                ProcessNodeEventData nodeData = JsonUtils.unmarshalFromByte(bytes, ProcessNodeEventData.class);
                if (nodeData.getStatus().isUnUsed()) {
                    // 跳过该process
                    continue;
                } else {
                    // select操作
                    currentStageStat.setStage(StageType.SELECT);
                    currentStageStat.setStartTime(zkProcessStat.getMtime());
                }
            } else {
                // 判断上一个节点，确定当前的stage
                StageType stage = prev.getStage();
                if (stage.isSelect()) {
                    currentStageStat.setStage(StageType.EXTRACT);
                } else if (stage.isExtract()) {
                    currentStageStat.setStage(StageType.TRANSFORM);
                } else if (stage.isTransform()) {
                    currentStageStat.setStage(StageType.LOAD);
                } else if (stage.isLoad()) {
                    // 已经是最后一个节点了
                    continue;
                }
                // 开始时间为上一个节点的结束时间
                currentStageStat.setStartTime(prev.getEndTime());
            }
            if (currentStageStat.getStage().isLoad()) {
                // load必须为第一个process节点
                if (i == 0) {
                    stageStats.add(currentStageStat);
                }
            } else {
                // 其他情况都添加
                stageStats.add(currentStageStat);
            }
        } catch (NoNodeException e) {
        // ignore
        } catch (KeeperException e) {
            throw new ArbitrateException(e);
        } catch (InterruptedException e) {
        // ignore
        } catch (UnsupportedEncodingException e) {
        // ignore
        }
        processStats.add(processStat);
    }
    return processStats;
}

Example 2 with StageType

use of com.alibaba.otter.shared.common.model.config.enums.StageType in project otter by alibaba.

the class NodeTaskServiceImpl method retain.

// 将target的目标除去source中的信息
private NodeTask retain(NodeTask targetTask, NodeTask sourceTask) {
    List<StageType> stages = targetTask.getStage();
    List<TaskEvent> events = targetTask.getEvent();
    List<StageType> mergeStates = new ArrayList<StageType>();
    List<TaskEvent> mergeEvents = new ArrayList<TaskEvent>();
    // 合并两者的交集的数据
    for (int i = 0; i < stages.size(); i++) {
        StageType stage = stages.get(i);
        TaskEvent event = events.get(i);
        // 找到source节点对应的TaskEvent
        TaskEvent sourceEvent = getMatchStage(sourceTask, stage);
        if (sourceEvent != null && sourceEvent != event) {
            // 存在相同的stage节点，判断event是否相同，不同则则添加
            mergeStates.add(stage);
            mergeEvents.add(event);
        }
    }
    // 添加targtTask中特有的stage/event
    for (int i = 0; i < stages.size(); i++) {
        StageType stage = stages.get(i);
        TaskEvent event = events.get(i);
        if (getMatchStage(sourceTask, stage) == null) {
            mergeStates.add(stage);
            mergeEvents.add(event);
        }
    }
    if (mergeStates.size() > 0) {
        NodeTask result = new NodeTask();
        result.setPipeline(targetTask.getPipeline());
        result.setEvent(mergeEvents);
        result.setStage(mergeStates);
        result.setShutdown(targetTask.isShutdown());
        return result;
    } else {
        return null;
    }
}

Example 3 with StageType

use of com.alibaba.otter.shared.common.model.config.enums.StageType in project otter by alibaba.

the class NodeTaskServiceImpl method merge.

// 合并两个NodeTask对象
private void merge(NodeTask target, NodeTask source) {
    List<StageType> stages = target.getStage();
    List<TaskEvent> events = target.getEvent();
    List<StageType> mergeStates = new ArrayList<StageType>();
    List<TaskEvent> mergeEvents = new ArrayList<TaskEvent>();
    // 合并两者的交集的数据
    for (int i = 0; i < stages.size(); i++) {
        StageType stage = stages.get(i);
        TaskEvent event = events.get(i);
        mergeStates.add(stage);
        // 找到source节点对应的TaskEvent，使用最新值
        TaskEvent sourceEvent = getMatchStage(source, stage);
        if (sourceEvent == null) {
            mergeEvents.add(event);
        } else {
            mergeEvents.add(sourceEvent);
        }
    }
    // 添加两者的差集，添加source中特有的节点
    List<StageType> sourceStages = source.getStage();
    List<TaskEvent> sourceEvents = source.getEvent();
    for (int i = 0; i < sourceStages.size(); i++) {
        StageType stage = sourceStages.get(i);
        TaskEvent event = sourceEvents.get(i);
        if (mergeStates.contains(stage)) {
            continue;
        }
        mergeStates.add(stage);
        mergeEvents.add(event);
    }
    // 更新一下数据
    target.setEvent(mergeEvents);
    target.setStage(mergeStates);
    // 更新下shutdown变量
    target.setShutdown(source.isShutdown());
}

Example 4 with StageType

use of com.alibaba.otter.shared.common.model.config.enums.StageType in project otter by alibaba.

the class OtterController method stop.

public void stop() throws Throwable {
    for (Map<StageType, GlobalTask> tasks : controllers.values()) {
        for (GlobalTask task : tasks.values()) {
            try {
                task.shutdown();
            } catch (Exception e) {
                logger.error("##shutdown task error!", e);
            }
        }
    }
    try {
        Long nid = configClientService.currentNode().getId();
        arbitrateManageService.nodeEvent().destory(Long.valueOf(nid));
    } catch (Exception e) {
        logger.error("##destory node error!", e);
    }
    try {
        arbitrateEventService.toolEvent().release();
    } catch (Exception e) {
        logger.error("##destory arbitrate error!", e);
    }
    try {
        // 通知manager停止当前node
        nodeTaskService.stopNode();
    } catch (Exception e) {
        logger.error("##stop node error!", e);
    }
    try {
        OtterContextLocator.close();
    } catch (Exception e) {
        logger.error("##cloes spring error!", e);
    }
    // 关闭zookeeper
    ZooKeeperClient.destory();
}

Example 5 with StageType

use of com.alibaba.otter.shared.common.model.config.enums.StageType in project otter by alibaba.

the class NodeTaskServiceImpl method parseNodeTask.

// 解析一下tasks为NodeTask对象
private List<NodeTask> parseNodeTask(Channel channel) {
    List<NodeTask> tasks = new ArrayList<NodeTask>();
    List<Pipeline> pipelines = channel.getPipelines();
    Long nid = configClientService.currentNode().getId();
    TaskEvent taksEvent = null;
    if (channel.getStatus().isStart()) {
        taksEvent = TaskEvent.CREATE;
    } else if (channel.getStatus().isStop()) {
        taksEvent = TaskEvent.DELETE;
    } else if (channel.getStatus().isPause()) {
        // modify by ljh at 2013-01-31 , pause状态也需要启动setl线程
        // 因为在发布的时候，restart指令不会推送指令，导致setl线程没有启动
        // return tasks;
        taksEvent = TaskEvent.CREATE;
    }
    // 处理当前最新的状态
    for (Pipeline pipeline : pipelines) {
        List<Node> sNodes = pipeline.getSelectNodes();
        for (Node node : sNodes) {
            if (nid.equals(node.getId())) {
                // 判断是否为当前的nid
                NodeTask task = new NodeTask();
                task.setPipeline(pipeline);
                NodeTask matchTask = getMatchTask(tasks, task);
                if (matchTask == null) {
                    matchTask = task;
                    tasks.add(task);
                }
                matchTask.setPipeline(pipeline);
                matchTask.getStage().add(StageType.SELECT);
                matchTask.getEvent().add(taksEvent);
            }
        }
        List<Node> eNodes = pipeline.getExtractNodes();
        for (Node node : eNodes) {
            if (nid.equals(node.getId())) {
                // 判断是否为当前的nid
                NodeTask task = new NodeTask();
                task.setPipeline(pipeline);
                NodeTask matchTask = getMatchTask(tasks, task);
                if (matchTask == null) {
                    matchTask = task;
                    tasks.add(task);
                }
                matchTask.getStage().add(StageType.EXTRACT);
                matchTask.getEvent().add(taksEvent);
            }
        }
        List<Node> tlNodes = pipeline.getLoadNodes();
        for (Node node : tlNodes) {
            if (nid.equals(node.getId())) {
                // 判断是否为当前的nid
                NodeTask task = new NodeTask();
                task.setPipeline(pipeline);
                NodeTask matchTask = getMatchTask(tasks, task);
                if (matchTask == null) {
                    matchTask = task;
                    tasks.add(task);
                }
                matchTask.getStage().add(StageType.TRANSFORM);
                matchTask.getEvent().add(taksEvent);
                matchTask.getStage().add(StageType.LOAD);
                matchTask.getEvent().add(taksEvent);
            }
        }
    }
    List<Long> pipelineIds = Lists.transform(channel.getPipelines(), new Function<Pipeline, Long>() {

        public Long apply(Pipeline input) {
            return input.getId();
        }
    });
    // 合并一下target中特有的记录，取一下反操作，表示要关闭
    for (NodeTask task : allTasks) {
        Pipeline pipeline = task.getPipeline();
        if (pipeline.getChannelId().equals(channel.getId()) && !pipelineIds.contains(pipeline.getId())) {
            // /是同一个channel，但对应的pipeline不在列表里
            // 处理pipeline删除
            NodeTask deletePipelineTask = new NodeTask();
            deletePipelineTask.setPipeline(pipeline);
            List<StageType> stages = task.getStage();
            List<TaskEvent> events = task.getEvent();
            for (int i = 0; i < stages.size(); i++) {
                StageType stage = stages.get(i);
                TaskEvent event = events.get(i);
                if (event.isCreate()) {
                    deletePipelineTask.getStage().add(stage);
                    // 添加为关闭
                    deletePipelineTask.getEvent().add(TaskEvent.DELETE);
                }
            }
            tasks.add(deletePipelineTask);
        }
        if (pipelineIds.contains(pipeline.getId())) {
            // 在当前的channel列表中
            boolean needAdd = false;
            // 找到对应的匹配
            NodeTask matchTask = getMatchTask(tasks, task);
            if (matchTask == null) {
                matchTask = new NodeTask();
                matchTask.setPipeline(pipeline);
                needAdd = true;
            }
            List<StageType> stages = task.getStage();
            List<TaskEvent> events = task.getEvent();
            for (int i = 0; i < stages.size(); i++) {
                StageType stage = stages.get(i);
                TaskEvent event = events.get(i);
                TaskEvent matchEvent = getMatchStage(matchTask, stage);
                if (matchEvent == null && event.isCreate()) {
                    // 对应的stage已经被移除，触发一个DELETE操作
                    matchTask.getStage().add(stage);
                    matchTask.getEvent().add(TaskEvent.DELETE);
                }
            }
            if (needAdd && matchTask.getStage().size() > 0) {
                tasks.add(matchTask);
            }
        }
    }
    // 判断当前的task是否需要全部关闭
    for (NodeTask task : tasks) {
        boolean shutdown = true;
        for (TaskEvent event : task.getEvent()) {
            // task已为当前最新节点信息
            shutdown &= event.isDelete();
        }
        task.setShutdown(shutdown);
    }
    return tasks;
}