use of io.cdap.cdap.api.workflow.WorkflowConditionNode in project cdap by caskdata.
the class DistributedWorkflowProgramRunner method findDriverResources.
/**
* Returns the {@link Resources} requirement for the workflow runnable deduced by Spark
* or MapReduce driver resources requirement.
*/
private Resources findDriverResources(Collection<WorkflowNode> nodes, Map<String, Resources> runnablesResources) {
// Find the resource requirements for the workflow based on the nodes memory requirements
Resources resources = new Resources();
for (WorkflowNode node : nodes) {
switch(node.getType()) {
case ACTION:
String programName = ((WorkflowActionNode) node).getProgram().getProgramName();
Resources runnableResources = runnablesResources.get(programName);
if (runnableResources != null) {
resources = maxResources(resources, runnableResources);
}
break;
case FORK:
Resources forkResources = ((WorkflowForkNode) node).getBranches().stream().map(branches -> findDriverResources(branches, runnablesResources)).reduce(this::mergeForkResources).orElse(resources);
resources = maxResources(resources, forkResources);
break;
case CONDITION:
Resources branchesResources = maxResources(findDriverResources(((WorkflowConditionNode) node).getIfBranch(), runnablesResources), findDriverResources(((WorkflowConditionNode) node).getElseBranch(), runnablesResources));
resources = maxResources(resources, branchesResources);
break;
default:
// This shouldn't happen unless we add new node type
LOG.warn("Ignoring unsupported Workflow node type {}", node.getType());
}
}
return resources;
}
use of io.cdap.cdap.api.workflow.WorkflowConditionNode in project cdap by caskdata.
the class WorkflowVerificationTest method verifyAnotherGoodWorkflowSpecification.
private void verifyAnotherGoodWorkflowSpecification(ApplicationSpecification appSpec) {
WorkflowSpecification spec = appSpec.getWorkflows().get("AnotherGoodWorkflow");
List<WorkflowNode> nodes = spec.getNodes();
Assert.assertTrue(nodes.size() == 4);
WorkflowNode node = nodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
WorkflowActionNode actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
node = nodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
WorkflowForkNode fork = (WorkflowForkNode) node;
Assert.assertTrue(fork.getBranches().size() == 2);
List<WorkflowNode> forkBranch1 = fork.getBranches().get(0);
Assert.assertTrue(forkBranch1.size() == 3);
node = forkBranch1.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR2")));
node = forkBranch1.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
WorkflowConditionNode condition = (WorkflowConditionNode) node;
Assert.assertTrue(condition.getPredicateClassName().contains("MyVerificationPredicate"));
List<WorkflowNode> ifNodes = condition.getIfBranch();
Assert.assertTrue(ifNodes.size() == 2);
List<WorkflowNode> elseNodes = condition.getElseBranch();
Assert.assertTrue(elseNodes.size() == 2);
node = ifNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR3")));
node = ifNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR4")));
node = elseNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR5")));
node = elseNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR6")));
node = forkBranch1.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR7")));
List<WorkflowNode> forkBranch2 = fork.getBranches().get(1);
Assert.assertTrue(forkBranch2.size() == 1);
node = forkBranch2.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR8")));
node = nodes.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
ifNodes = ((WorkflowConditionNode) node).getIfBranch();
elseNodes = ((WorkflowConditionNode) node).getElseBranch();
Assert.assertTrue(ifNodes.size() == 2);
node = ifNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
node = ifNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP2")));
Assert.assertTrue(elseNodes.size() == 3);
node = elseNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP3")));
node = elseNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP4")));
node = elseNodes.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
WorkflowForkNode anotherFork = (WorkflowForkNode) node;
Assert.assertTrue(anotherFork.getBranches().size() == 2);
List<WorkflowNode> anotherForkBranch1 = anotherFork.getBranches().get(0);
Assert.assertTrue(anotherForkBranch1.size() == 1);
List<WorkflowNode> anotherForkBranch2 = anotherFork.getBranches().get(1);
Assert.assertTrue(anotherForkBranch2.size() == 1);
node = anotherForkBranch1.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP5")));
node = anotherForkBranch2.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP6")));
node = nodes.get(3);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP7")));
}
use of io.cdap.cdap.api.workflow.WorkflowConditionNode in project cdap by caskdata.
the class DefaultWorkflowConfigurer method addWorkflowConditionNode.
@Override
public void addWorkflowConditionNode(Condition condition, List<WorkflowNode> ifBranch, List<WorkflowNode> elseBranch) {
Preconditions.checkArgument(condition != null, "Condition is null.");
ConditionSpecification spec = DefaultConditionConfigurer.configureCondition(condition, deployNamespace, artifactId, pluginFinder, pluginInstantiator, runtimeInfo, getFeatureFlagsProvider());
nodes.add(new WorkflowConditionNode(spec.getName(), spec, ifBranch, elseBranch));
}
use of io.cdap.cdap.api.workflow.WorkflowConditionNode in project cdap by caskdata.
the class DefaultWorkflowConfigurer method createConditionNodeWithId.
private WorkflowNode createConditionNodeWithId(WorkflowNode node) {
WorkflowConditionNode conditionNode = (WorkflowConditionNode) node;
List<WorkflowNode> ifbranch = Lists.newArrayList();
List<WorkflowNode> elsebranch = Lists.newArrayList();
ifbranch.addAll(createNodesWithId(conditionNode.getIfBranch()));
elsebranch.addAll(createNodesWithId(conditionNode.getElseBranch()));
ConditionSpecification spec = conditionNode.getConditionSpecification();
return new WorkflowConditionNode(spec.getName(), spec, ifbranch, elsebranch);
}
use of io.cdap.cdap.api.workflow.WorkflowConditionNode in project cdap by caskdata.
the class DefaultWorkflowForkConfigurer method addWorkflowConditionNode.
@Override
public void addWorkflowConditionNode(Condition condition, List<WorkflowNode> ifBranch, List<WorkflowNode> elseBranch) {
Preconditions.checkArgument(condition != null, "Condition is null.");
ConditionSpecification spec = DefaultConditionConfigurer.configureCondition(condition, deployNamespace, artifactId, pluginFinder, pluginInstantiator, runtimeInfo, featureFlagsProvider);
currentBranch.add(new WorkflowConditionNode(spec.getName(), spec, ifBranch, elseBranch));
}
Aggregations