Examples with WorkflowNode io.cdap.cdap.api.workflow.WorkflowNode used on opensource projects

Example 16 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by cdapio.

the class DefaultWorkflowConfigurer method createForkNodeWithId.

private WorkflowNode createForkNodeWithId(WorkflowNode node) {
    String forkNodeId = Integer.toString(nodeIdentifier++);
    List<List<WorkflowNode>> branches = Lists.newArrayList();
    WorkflowForkNode forkNode = (WorkflowForkNode) node;
    for (List<WorkflowNode> branch : forkNode.getBranches()) {
        branches.add(createNodesWithId(branch));
    }
    return new WorkflowForkNode(forkNodeId, branches);
}

Example 17 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by cdapio.

the class WorkflowVerificationTest method verifyGoodWorkflowSpecifications.

private void verifyGoodWorkflowSpecifications(ApplicationSpecification appSpec) {
    WorkflowSpecification spec = appSpec.getWorkflows().get("GoodWorkflow");
    Assert.assertTrue(spec.getNodes().size() == 4);
    List<WorkflowNode> nodes = spec.getNodes();
    WorkflowNode node = nodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    WorkflowActionNode actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    node = nodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
    WorkflowForkNode fork1 = (WorkflowForkNode) node;
    Assert.assertTrue(fork1.getBranches().size() == 2);
    List<WorkflowNode> fork1Branch1 = fork1.getBranches().get(0);
    Assert.assertTrue(fork1Branch1.size() == 2);
    List<WorkflowNode> fork1Branch2 = fork1.getBranches().get(1);
    Assert.assertTrue(fork1Branch2.size() == 1);
    actionNode = (WorkflowActionNode) fork1Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    WorkflowForkNode fork2 = (WorkflowForkNode) fork1Branch1.get(1);
    List<WorkflowNode> fork2Branch1 = fork2.getBranches().get(0);
    Assert.assertTrue(fork2Branch1.size() == 2);
    List<WorkflowNode> fork2Branch2 = fork2.getBranches().get(1);
    Assert.assertTrue(fork2Branch2.size() == 1);
    actionNode = (WorkflowActionNode) fork2Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    WorkflowForkNode fork3 = (WorkflowForkNode) fork2Branch1.get(1);
    List<WorkflowNode> fork3Branch1 = fork3.getBranches().get(0);
    Assert.assertTrue(fork3Branch1.size() == 2);
    List<WorkflowNode> fork3Branch2 = fork3.getBranches().get(1);
    Assert.assertTrue(fork3Branch2.size() == 1);
    WorkflowForkNode fork4 = (WorkflowForkNode) fork3Branch1.get(0);
    List<WorkflowNode> fork4Branch1 = fork4.getBranches().get(0);
    Assert.assertTrue(fork4Branch1.size() == 2);
    List<WorkflowNode> fork4Branch2 = fork4.getBranches().get(1);
    Assert.assertTrue(fork4Branch2.size() == 1);
    actionNode = (WorkflowActionNode) fork4Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    actionNode = (WorkflowActionNode) fork4Branch1.get(1);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork4Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork4Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork4Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork2Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork1Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    node = nodes.get(2);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
    node = nodes.get(3);
    WorkflowForkNode fork5 = (WorkflowForkNode) node;
    List<WorkflowNode> fork5Branch1 = fork5.getBranches().get(0);
    Assert.assertTrue(fork5Branch1.size() == 1);
    List<WorkflowNode> fork5Branch2 = fork5.getBranches().get(1);
    Assert.assertTrue(fork5Branch2.size() == 1);
    actionNode = (WorkflowActionNode) fork5Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork5Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
}

Example 18 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by cdapio.

the class DistributedWorkflowProgramRunner method validateOptions.

@Override
protected void validateOptions(Program program, ProgramOptions options) {
    super.validateOptions(program, options);
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.WORKFLOW, "Only WORKFLOW process type is supported.");
    WorkflowSpecification spec = appSpec.getWorkflows().get(program.getName());
    Preconditions.checkNotNull(spec, "Missing WorkflowSpecification for %s", program.getName());
    for (WorkflowNode node : spec.getNodes()) {
        if (node.getType().equals(WorkflowNodeType.ACTION)) {
            SystemArguments.validateTransactionTimeout(options.getUserArguments().asMap(), cConf, "action", node.getNodeId());
        }
    }
}

Example 19 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by cdapio.

the class DistributedWorkflowProgramRunner method findDriverResources.

/**
 * Returns the {@link Resources} requirement for the workflow runnable deduced by Spark
 * or MapReduce driver resources requirement.
 */
private Resources findDriverResources(Collection<WorkflowNode> nodes, Map<String, Resources> runnablesResources) {
    // Find the resource requirements for the workflow based on the nodes memory requirements
    Resources resources = new Resources();
    for (WorkflowNode node : nodes) {
        switch(node.getType()) {
            case ACTION:
                String programName = ((WorkflowActionNode) node).getProgram().getProgramName();
                Resources runnableResources = runnablesResources.get(programName);
                if (runnableResources != null) {
                    resources = maxResources(resources, runnableResources);
                }
                break;
            case FORK:
                Resources forkResources = ((WorkflowForkNode) node).getBranches().stream().map(branches -> findDriverResources(branches, runnablesResources)).reduce(this::mergeForkResources).orElse(resources);
                resources = maxResources(resources, forkResources);
                break;
            case CONDITION:
                Resources branchesResources = maxResources(findDriverResources(((WorkflowConditionNode) node).getIfBranch(), runnablesResources), findDriverResources(((WorkflowConditionNode) node).getElseBranch(), runnablesResources));
                resources = maxResources(resources, branchesResources);
                break;
            default:
                // This shouldn't happen unless we add new node type
                LOG.warn("Ignoring unsupported Workflow node type {}", node.getType());
        }
    }
    return resources;
}

Example 20 with WorkflowNode

use of io.cdap.cdap.api.workflow.WorkflowNode in project cdap by cdapio.

the class LineageAdmin method extractAndAddInnerPrograms.

/**
 * Extract inner programs and runs from the workflow run record, the run record's properties have all the
 * inner program run ids. The workflow spec can then be used to determine what the inner programs are and
 * create the program run ids for them
 */
private void extractAndAddInnerPrograms(Set<ProgramId> toVisitPrograms, Map<ProgramRunId, ProgramRunId> programWorkflowMap, Map<ApplicationId, ApplicationSpecification> appSpecs, ProgramRunId programRunId, RunRecordDetail wfRunRecord) {
    ApplicationId appId = programRunId.getParent().getParent();
    WorkflowSpecification workflowSpec = appSpecs.get(appId).getWorkflows().get(programRunId.getProgram());
    Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap();
    wfRunRecord.getProperties().forEach((key, value) -> {
        if (nodeIdMap.containsKey(key)) {
            WorkflowActionNode node = (WorkflowActionNode) nodeIdMap.get(key);
            ProgramType type = ProgramType.valueOf(node.getProgram().getProgramType().name());
            ProgramId program = appId.program(type, key);
            programWorkflowMap.put(program.run(value), programRunId);
            toVisitPrograms.add(program);
        }
    });
}