Examples with WorkflowNode io.cdap.cdap.api.workflow.WorkflowNode used on opensource projects

Example 1 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class DistributedWorkflowProgramRunner method findDriverResources.

/**
 * Returns the {@link Resources} requirement for the workflow runnable deduced by Spark
 * or MapReduce driver resources requirement.
 */
private Resources findDriverResources(Collection<WorkflowNode> nodes, Map<String, Resources> runnablesResources) {
    // Find the resource requirements for the workflow based on the nodes memory requirements
    Resources resources = new Resources();
    for (WorkflowNode node : nodes) {
        switch(node.getType()) {
            case ACTION:
                String programName = ((WorkflowActionNode) node).getProgram().getProgramName();
                Resources runnableResources = runnablesResources.get(programName);
                if (runnableResources != null) {
                    resources = maxResources(resources, runnableResources);
                }
                break;
            case FORK:
                Resources forkResources = ((WorkflowForkNode) node).getBranches().stream().map(branches -> findDriverResources(branches, runnablesResources)).reduce(this::mergeForkResources).orElse(resources);
                resources = maxResources(resources, forkResources);
                break;
            case CONDITION:
                Resources branchesResources = maxResources(findDriverResources(((WorkflowConditionNode) node).getIfBranch(), runnablesResources), findDriverResources(((WorkflowConditionNode) node).getElseBranch(), runnablesResources));
                resources = maxResources(resources, branchesResources);
                break;
            default:
                // This shouldn't happen unless we add new node type
                LOG.warn("Ignoring unsupported Workflow node type {}", node.getType());
        }
    }
    return resources;
}

Example 2 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class WorkflowVerificationTest method verifyWorkflowWithLocalDatasetSpecification.

private void verifyWorkflowWithLocalDatasetSpecification(ApplicationSpecification appSpec) {
    WorkflowSpecification spec = appSpec.getWorkflows().get("WorkflowWithLocalDatasets");
    List<WorkflowNode> nodes = spec.getNodes();
    Assert.assertTrue(nodes.size() == 2);
    WorkflowNode node = nodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    WorkflowActionNode actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
    node = nodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
    Map<String, DatasetCreationSpec> localDatasetSpecs = spec.getLocalDatasetSpecs();
    Assert.assertEquals(5, localDatasetSpecs.size());
    DatasetCreationSpec datasetCreationSpec = localDatasetSpecs.get("mytable");
    Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfile");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfile_with_properties");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals("prop_value", datasetCreationSpec.getProperties().getProperties().get("prop_key"));
    datasetCreationSpec = localDatasetSpecs.get("mytablefromtype");
    Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfilefromtype");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals("another_prop_value", datasetCreationSpec.getProperties().getProperties().get("another_prop_key"));
    // Check if the application specification has correct modules
    Map<String, String> datasetModules = appSpec.getDatasetModules();
    Assert.assertEquals(2, datasetModules.size());
    Assert.assertTrue(datasetModules.containsKey(FileSet.class.getName()));
    Assert.assertTrue(datasetModules.containsKey(Table.class.getName()));
}

Example 3 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class WorkflowVerificationTest method verifyAnotherGoodWorkflowSpecification.

private void verifyAnotherGoodWorkflowSpecification(ApplicationSpecification appSpec) {
    WorkflowSpecification spec = appSpec.getWorkflows().get("AnotherGoodWorkflow");
    List<WorkflowNode> nodes = spec.getNodes();
    Assert.assertTrue(nodes.size() == 4);
    WorkflowNode node = nodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    WorkflowActionNode actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
    node = nodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
    WorkflowForkNode fork = (WorkflowForkNode) node;
    Assert.assertTrue(fork.getBranches().size() == 2);
    List<WorkflowNode> forkBranch1 = fork.getBranches().get(0);
    Assert.assertTrue(forkBranch1.size() == 3);
    node = forkBranch1.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR2")));
    node = forkBranch1.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
    WorkflowConditionNode condition = (WorkflowConditionNode) node;
    Assert.assertTrue(condition.getPredicateClassName().contains("MyVerificationPredicate"));
    List<WorkflowNode> ifNodes = condition.getIfBranch();
    Assert.assertTrue(ifNodes.size() == 2);
    List<WorkflowNode> elseNodes = condition.getElseBranch();
    Assert.assertTrue(elseNodes.size() == 2);
    node = ifNodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR3")));
    node = ifNodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR4")));
    node = elseNodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR5")));
    node = elseNodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR6")));
    node = forkBranch1.get(2);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR7")));
    List<WorkflowNode> forkBranch2 = fork.getBranches().get(1);
    Assert.assertTrue(forkBranch2.size() == 1);
    node = forkBranch2.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR8")));
    node = nodes.get(2);
    Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
    ifNodes = ((WorkflowConditionNode) node).getIfBranch();
    elseNodes = ((WorkflowConditionNode) node).getElseBranch();
    Assert.assertTrue(ifNodes.size() == 2);
    node = ifNodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
    node = ifNodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP2")));
    Assert.assertTrue(elseNodes.size() == 3);
    node = elseNodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP3")));
    node = elseNodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP4")));
    node = elseNodes.get(2);
    Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
    WorkflowForkNode anotherFork = (WorkflowForkNode) node;
    Assert.assertTrue(anotherFork.getBranches().size() == 2);
    List<WorkflowNode> anotherForkBranch1 = anotherFork.getBranches().get(0);
    Assert.assertTrue(anotherForkBranch1.size() == 1);
    List<WorkflowNode> anotherForkBranch2 = anotherFork.getBranches().get(1);
    Assert.assertTrue(anotherForkBranch2.size() == 1);
    node = anotherForkBranch1.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP5")));
    node = anotherForkBranch2.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP6")));
    node = nodes.get(3);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP7")));
}

Example 4 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class LineageAdmin method extractAndAddInnerPrograms.

/**
 * Extract inner programs and runs from the workflow run record, the run record's properties have all the
 * inner program run ids. The workflow spec can then be used to determine what the inner programs are and
 * create the program run ids for them
 */
private void extractAndAddInnerPrograms(Set<ProgramId> toVisitPrograms, Map<ProgramRunId, ProgramRunId> programWorkflowMap, Map<ApplicationId, ApplicationSpecification> appSpecs, ProgramRunId programRunId, RunRecordDetail wfRunRecord) {
    ApplicationId appId = programRunId.getParent().getParent();
    WorkflowSpecification workflowSpec = appSpecs.get(appId).getWorkflows().get(programRunId.getProgram());
    Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap();
    wfRunRecord.getProperties().forEach((key, value) -> {
        if (nodeIdMap.containsKey(key)) {
            WorkflowActionNode node = (WorkflowActionNode) nodeIdMap.get(key);
            ProgramType type = ProgramType.valueOf(node.getProgram().getProgramType().name());
            ProgramId program = appId.program(type, key);
            programWorkflowMap.put(program.run(value), programRunId);
            toVisitPrograms.add(program);
        }
    });
}

Example 5 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class DefaultWorkflowConfigurer method createForkNodeWithId.

private WorkflowNode createForkNodeWithId(WorkflowNode node) {
    String forkNodeId = Integer.toString(nodeIdentifier++);
    List<List<WorkflowNode>> branches = Lists.newArrayList();
    WorkflowForkNode forkNode = (WorkflowForkNode) node;
    for (List<WorkflowNode> branch : forkNode.getBranches()) {
        branches.add(createNodesWithId(branch));
    }
    return new WorkflowForkNode(forkNodeId, branches);
}