Examples with TaskNodeRelation org.apache.dolphinscheduler.common.model.TaskNodeRelation used on opensource projects

Example 1 with TaskNodeRelation

use of org.apache.dolphinscheduler.common.model.TaskNodeRelation in project dolphinscheduler by apache.

the class ProcessDefinitionService method viewTree.

/**
 * Encapsulates the TreeView structure
 *
 * @param processId process definition id
 * @param limit limit
 * @return tree view json data
 * @throws Exception exception
 */
public Map<String, Object> viewTree(Integer processId, Integer limit) throws Exception {
    Map<String, Object> result = new HashMap<>();
    ProcessDefinition processDefinition = processDefineMapper.selectById(processId);
    if (null == processDefinition) {
        logger.info("process define not exists");
        putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinition);
        return result;
    }
    DAG<String, TaskNode, TaskNodeRelation> dag = genDagGraph(processDefinition);
    /**
     * nodes that is running
     */
    Map<String, List<TreeViewDto>> runningNodeMap = new ConcurrentHashMap<>();
    /**
     * nodes that is waiting torun
     */
    Map<String, List<TreeViewDto>> waitingRunningNodeMap = new ConcurrentHashMap<>();
    /**
     * List of process instances
     */
    List<ProcessInstance> processInstanceList = processInstanceMapper.queryByProcessDefineId(processId, limit);
    for (ProcessInstance processInstance : processInstanceList) {
        processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime()));
    }
    if (limit > processInstanceList.size()) {
        limit = processInstanceList.size();
    }
    TreeViewDto parentTreeViewDto = new TreeViewDto();
    parentTreeViewDto.setName("DAG");
    parentTreeViewDto.setType("");
    for (int i = limit - 1; i >= 0; i--) {
        ProcessInstance processInstance = processInstanceList.get(i);
        Date endTime = processInstance.getEndTime() == null ? new Date() : processInstance.getEndTime();
        parentTreeViewDto.getInstances().add(new Instance(processInstance.getId(), processInstance.getName(), "", processInstance.getState().toString(), processInstance.getStartTime(), endTime, processInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - processInstance.getStartTime().getTime())));
    }
    List<TreeViewDto> parentTreeViewDtoList = new ArrayList<>();
    parentTreeViewDtoList.add(parentTreeViewDto);
    // Here is the encapsulation task instance
    for (String startNode : dag.getBeginNode()) {
        runningNodeMap.put(startNode, parentTreeViewDtoList);
    }
    while (Stopper.isRunning()) {
        Set<String> postNodeList = null;
        Iterator<Map.Entry<String, List<TreeViewDto>>> iter = runningNodeMap.entrySet().iterator();
        while (iter.hasNext()) {
            Map.Entry<String, List<TreeViewDto>> en = iter.next();
            String nodeName = en.getKey();
            parentTreeViewDtoList = en.getValue();
            TreeViewDto treeViewDto = new TreeViewDto();
            treeViewDto.setName(nodeName);
            TaskNode taskNode = dag.getNode(nodeName);
            treeViewDto.setType(taskNode.getType());
            // set treeViewDto instances
            for (int i = limit - 1; i >= 0; i--) {
                ProcessInstance processInstance = processInstanceList.get(i);
                TaskInstance taskInstance = taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), nodeName);
                if (taskInstance == null) {
                    treeViewDto.getInstances().add(new Instance(-1, "not running", null));
                } else {
                    Date startTime = taskInstance.getStartTime() == null ? new Date() : taskInstance.getStartTime();
                    Date endTime = taskInstance.getEndTime() == null ? new Date() : taskInstance.getEndTime();
                    int subProcessId = 0;
                    /**
                     * if process is sub process, the return sub id, or sub id=0
                     */
                    if (taskInstance.getTaskType().equals(TaskType.SUB_PROCESS.name())) {
                        String taskJson = taskInstance.getTaskJson();
                        taskNode = JSON.parseObject(taskJson, TaskNode.class);
                        subProcessId = Integer.parseInt(JSON.parseObject(taskNode.getParams()).getString(CMDPARAM_SUB_PROCESS_DEFINE_ID));
                    }
                    treeViewDto.getInstances().add(new Instance(taskInstance.getId(), taskInstance.getName(), taskInstance.getTaskType(), taskInstance.getState().toString(), taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - startTime.getTime()), subProcessId));
                }
            }
            for (TreeViewDto pTreeViewDto : parentTreeViewDtoList) {
                pTreeViewDto.getChildren().add(treeViewDto);
            }
            postNodeList = dag.getSubsequentNodes(nodeName);
            if (CollectionUtils.isNotEmpty(postNodeList)) {
                for (String nextNodeName : postNodeList) {
                    List<TreeViewDto> treeViewDtoList = waitingRunningNodeMap.get(nextNodeName);
                    if (CollectionUtils.isNotEmpty(treeViewDtoList)) {
                        treeViewDtoList.add(treeViewDto);
                        waitingRunningNodeMap.put(nextNodeName, treeViewDtoList);
                    } else {
                        treeViewDtoList = new ArrayList<>();
                        treeViewDtoList.add(treeViewDto);
                        waitingRunningNodeMap.put(nextNodeName, treeViewDtoList);
                    }
                }
            }
            runningNodeMap.remove(nodeName);
        }
        if (waitingRunningNodeMap == null || waitingRunningNodeMap.size() == 0) {
            break;
        } else {
            runningNodeMap.putAll(waitingRunningNodeMap);
            waitingRunningNodeMap.clear();
        }
    }
    result.put(Constants.DATA_LIST, parentTreeViewDto);
    result.put(Constants.STATUS, Status.SUCCESS);
    result.put(Constants.MSG, Status.SUCCESS.getMsg());
    return result;
}

Example 2 with TaskNodeRelation

use of org.apache.dolphinscheduler.common.model.TaskNodeRelation in project dolphinscheduler by apache.

the class DagHelper method generateRelationListByFlowNodes.

/**
 * generate flow node relation list by task node list;
 * Edges that are not in the task Node List will not be added to the result
 * @param taskNodeList taskNodeList
 * @return task node relation list
 */
public static List<TaskNodeRelation> generateRelationListByFlowNodes(List<TaskNode> taskNodeList) {
    List<TaskNodeRelation> nodeRelationList = new ArrayList<>();
    for (TaskNode taskNode : taskNodeList) {
        String preTasks = taskNode.getPreTasks();
        List<String> preTaskList = JSONUtils.toList(preTasks, String.class);
        if (preTaskList != null) {
            for (String depNodeName : preTaskList) {
                if (null != findNodeByName(taskNodeList, depNodeName)) {
                    nodeRelationList.add(new TaskNodeRelation(depNodeName, taskNode.getName()));
                }
            }
        }
    }
    return nodeRelationList;
}

Example 3 with TaskNodeRelation

use of org.apache.dolphinscheduler.common.model.TaskNodeRelation in project dolphinscheduler by apache.

the class DagHelperTest method testForbiddenPostNode.

/**
 * test forbidden post node
 * @throws JsonProcessingException
 */
@Test
public void testForbiddenPostNode() throws JsonProcessingException {
    DAG<String, TaskNode, TaskNodeRelation> dag = generateDag();
    Map<String, TaskInstance> completeTaskList = new HashMap<>();
    Map<String, TaskNode> skipNodeList = new HashMap<>();
    Set<String> postNodes = null;
    // dag: 1-2-3-5-7 4-3-6 2-8-5-7
    // forbid:2 complete:1  post:4/8
    completeTaskList.put("1", new TaskInstance());
    TaskNode node2 = dag.getNode("2");
    node2.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
    postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList);
    Assert.assertEquals(2, postNodes.size());
    Assert.assertTrue(postNodes.contains("4"));
    Assert.assertTrue(postNodes.contains("8"));
    // forbid:2/4 complete:1 post:3/8
    TaskNode node4 = dag.getNode("4");
    node4.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
    postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList);
    Assert.assertEquals(2, postNodes.size());
    Assert.assertTrue(postNodes.contains("3"));
    Assert.assertTrue(postNodes.contains("8"));
    // forbid:2/4/5 complete:1/8 post:3
    completeTaskList.put("8", new TaskInstance());
    TaskNode node5 = dag.getNode("5");
    node5.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
    postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList);
    Assert.assertEquals(1, postNodes.size());
    Assert.assertTrue(postNodes.contains("3"));
}

Example 4 with TaskNodeRelation

use of org.apache.dolphinscheduler.common.model.TaskNodeRelation in project dolphinscheduler by apache.

the class DagHelperTest method generateDag.

/**
 * 1->2->3->5->7
 * 4->3->6
 * 2->8->5->7
 *
 * @return dag
 * @throws JsonProcessingException if error throws JsonProcessingException
 */
private DAG<String, TaskNode, TaskNodeRelation> generateDag() throws JsonProcessingException {
    List<TaskNode> taskNodeList = new ArrayList<>();
    TaskNode node1 = new TaskNode();
    node1.setId("1");
    node1.setName("1");
    node1.setType("SHELL");
    taskNodeList.add(node1);
    TaskNode node2 = new TaskNode();
    node2.setId("2");
    node2.setName("2");
    node2.setType("SHELL");
    List<String> dep2 = new ArrayList<>();
    dep2.add("1");
    node2.setDepList(dep2);
    taskNodeList.add(node2);
    TaskNode node4 = new TaskNode();
    node4.setId("4");
    node4.setName("4");
    node4.setType("SHELL");
    taskNodeList.add(node4);
    TaskNode node3 = new TaskNode();
    node3.setId("3");
    node3.setName("3");
    node3.setType("SHELL");
    List<String> dep3 = new ArrayList<>();
    dep3.add("2");
    dep3.add("4");
    node3.setDepList(dep3);
    taskNodeList.add(node3);
    TaskNode node5 = new TaskNode();
    node5.setId("5");
    node5.setName("5");
    node5.setType("SHELL");
    List<String> dep5 = new ArrayList<>();
    dep5.add("3");
    dep5.add("8");
    node5.setDepList(dep5);
    taskNodeList.add(node5);
    TaskNode node6 = new TaskNode();
    node6.setId("6");
    node6.setName("6");
    node6.setType("SHELL");
    List<String> dep6 = new ArrayList<>();
    dep6.add("3");
    node6.setDepList(dep6);
    taskNodeList.add(node6);
    TaskNode node7 = new TaskNode();
    node7.setId("7");
    node7.setName("7");
    node7.setType("SHELL");
    List<String> dep7 = new ArrayList<>();
    dep7.add("5");
    node7.setDepList(dep7);
    taskNodeList.add(node7);
    TaskNode node8 = new TaskNode();
    node8.setId("8");
    node8.setName("8");
    node8.setType("SHELL");
    List<String> dep8 = new ArrayList<>();
    dep8.add("2");
    node8.setDepList(dep8);
    taskNodeList.add(node8);
    List<String> startNodes = new ArrayList<>();
    List<String> recoveryNodes = new ArrayList<>();
    List<TaskNode> destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList, startNodes, recoveryNodes, TaskDependType.TASK_POST);
    List<TaskNodeRelation> taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList);
    ProcessDag processDag = new ProcessDag();
    processDag.setEdges(taskNodeRelations);
    processDag.setNodes(destTaskNodeList);
    return DagHelper.buildDagGraph(processDag);
}

Example 5 with TaskNodeRelation

use of org.apache.dolphinscheduler.common.model.TaskNodeRelation in project dolphinscheduler by apache.

the class DagHelperTest method testTaskNodeCanSubmit.

/**
 * test task node can submit
 *
 * @throws JsonProcessingException if error throws JsonProcessingException
 */
@Test
public void testTaskNodeCanSubmit() throws JsonProcessingException {
    // 1->2->3->5->7
    // 4->3->6
    DAG<String, TaskNode, TaskNodeRelation> dag = generateDag();
    TaskNode taskNode3 = dag.getNode("3");
    Map<String, TaskInstance> completeTaskList = new HashMap<>();
    Map<String, TaskNode> skipNodeList = new HashMap<>();
    completeTaskList.putIfAbsent("1", new TaskInstance());
    Boolean canSubmit = false;
    // 2/4 are forbidden submit 3
    TaskNode node2 = dag.getNode("2");
    node2.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
    TaskNode nodex = dag.getNode("4");
    nodex.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
    canSubmit = DagHelper.allDependsForbiddenOrEnd(taskNode3, dag, skipNodeList, completeTaskList);
    Assert.assertEquals(canSubmit, true);
    // 2forbidden, 3 cannot be submit
    completeTaskList.putIfAbsent("2", new TaskInstance());
    TaskNode nodey = dag.getNode("4");
    nodey.setRunFlag("");
    canSubmit = DagHelper.allDependsForbiddenOrEnd(taskNode3, dag, skipNodeList, completeTaskList);
    Assert.assertEquals(canSubmit, false);
    // 2/3 forbidden submit 5
    TaskNode node3 = dag.getNode("3");
    node3.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
    TaskNode node8 = dag.getNode("8");
    node8.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
    TaskNode node5 = dag.getNode("5");
    canSubmit = DagHelper.allDependsForbiddenOrEnd(node5, dag, skipNodeList, completeTaskList);
    Assert.assertEquals(canSubmit, true);
}