use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by caskdata.
the class WorkflowVerificationTest method verifyWorkflowWithLocalDatasetSpecification.
private void verifyWorkflowWithLocalDatasetSpecification(ApplicationSpecification appSpec) {
WorkflowSpecification spec = appSpec.getWorkflows().get("WorkflowWithLocalDatasets");
List<WorkflowNode> nodes = spec.getNodes();
Assert.assertTrue(nodes.size() == 2);
WorkflowNode node = nodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
WorkflowActionNode actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
node = nodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
Map<String, DatasetCreationSpec> localDatasetSpecs = spec.getLocalDatasetSpecs();
Assert.assertEquals(5, localDatasetSpecs.size());
DatasetCreationSpec datasetCreationSpec = localDatasetSpecs.get("mytable");
Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
datasetCreationSpec = localDatasetSpecs.get("myfile");
Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
datasetCreationSpec = localDatasetSpecs.get("myfile_with_properties");
Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals("prop_value", datasetCreationSpec.getProperties().getProperties().get("prop_key"));
datasetCreationSpec = localDatasetSpecs.get("mytablefromtype");
Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
datasetCreationSpec = localDatasetSpecs.get("myfilefromtype");
Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals("another_prop_value", datasetCreationSpec.getProperties().getProperties().get("another_prop_key"));
// Check if the application specification has correct modules
Map<String, String> datasetModules = appSpec.getDatasetModules();
Assert.assertEquals(2, datasetModules.size());
Assert.assertTrue(datasetModules.containsKey(FileSet.class.getName()));
Assert.assertTrue(datasetModules.containsKey(Table.class.getName()));
}
use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by caskdata.
the class WorkflowVerificationTest method verifyAnotherGoodWorkflowSpecification.
private void verifyAnotherGoodWorkflowSpecification(ApplicationSpecification appSpec) {
WorkflowSpecification spec = appSpec.getWorkflows().get("AnotherGoodWorkflow");
List<WorkflowNode> nodes = spec.getNodes();
Assert.assertTrue(nodes.size() == 4);
WorkflowNode node = nodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
WorkflowActionNode actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
node = nodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
WorkflowForkNode fork = (WorkflowForkNode) node;
Assert.assertTrue(fork.getBranches().size() == 2);
List<WorkflowNode> forkBranch1 = fork.getBranches().get(0);
Assert.assertTrue(forkBranch1.size() == 3);
node = forkBranch1.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR2")));
node = forkBranch1.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
WorkflowConditionNode condition = (WorkflowConditionNode) node;
Assert.assertTrue(condition.getPredicateClassName().contains("MyVerificationPredicate"));
List<WorkflowNode> ifNodes = condition.getIfBranch();
Assert.assertTrue(ifNodes.size() == 2);
List<WorkflowNode> elseNodes = condition.getElseBranch();
Assert.assertTrue(elseNodes.size() == 2);
node = ifNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR3")));
node = ifNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR4")));
node = elseNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR5")));
node = elseNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR6")));
node = forkBranch1.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR7")));
List<WorkflowNode> forkBranch2 = fork.getBranches().get(1);
Assert.assertTrue(forkBranch2.size() == 1);
node = forkBranch2.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR8")));
node = nodes.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
ifNodes = ((WorkflowConditionNode) node).getIfBranch();
elseNodes = ((WorkflowConditionNode) node).getElseBranch();
Assert.assertTrue(ifNodes.size() == 2);
node = ifNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
node = ifNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP2")));
Assert.assertTrue(elseNodes.size() == 3);
node = elseNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP3")));
node = elseNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP4")));
node = elseNodes.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
WorkflowForkNode anotherFork = (WorkflowForkNode) node;
Assert.assertTrue(anotherFork.getBranches().size() == 2);
List<WorkflowNode> anotherForkBranch1 = anotherFork.getBranches().get(0);
Assert.assertTrue(anotherForkBranch1.size() == 1);
List<WorkflowNode> anotherForkBranch2 = anotherFork.getBranches().get(1);
Assert.assertTrue(anotherForkBranch2.size() == 1);
node = anotherForkBranch1.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP5")));
node = anotherForkBranch2.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP6")));
node = nodes.get(3);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP7")));
}
use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by caskdata.
the class LineageAdmin method extractAndAddInnerPrograms.
/**
* Extract inner programs and runs from the workflow run record, the run record's properties have all the
* inner program run ids. The workflow spec can then be used to determine what the inner programs are and
* create the program run ids for them
*/
private void extractAndAddInnerPrograms(Set<ProgramId> toVisitPrograms, Map<ProgramRunId, ProgramRunId> programWorkflowMap, Map<ApplicationId, ApplicationSpecification> appSpecs, ProgramRunId programRunId, RunRecordDetail wfRunRecord) {
ApplicationId appId = programRunId.getParent().getParent();
WorkflowSpecification workflowSpec = appSpecs.get(appId).getWorkflows().get(programRunId.getProgram());
Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap();
wfRunRecord.getProperties().forEach((key, value) -> {
if (nodeIdMap.containsKey(key)) {
WorkflowActionNode node = (WorkflowActionNode) nodeIdMap.get(key);
ProgramType type = ProgramType.valueOf(node.getProgram().getProgramType().name());
ProgramId program = appId.program(type, key);
programWorkflowMap.put(program.run(value), programRunId);
toVisitPrograms.add(program);
}
});
}
use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by caskdata.
the class WorkflowNodeCreator method createWorkflowCustomActionNode.
static WorkflowNode createWorkflowCustomActionNode(CustomAction action, Id.Namespace deployNamespace, Id.Artifact artifactId, PluginFinder pluginFinder, PluginInstantiator pluginInstantiator, @Nullable AppDeploymentRuntimeInfo runtimeInfo, FeatureFlagsProvider featureFlagsProvider) {
Preconditions.checkArgument(action != null, "CustomAction is null.");
CustomActionSpecification spec = DefaultCustomActionConfigurer.configureAction(action, deployNamespace, artifactId, pluginFinder, pluginInstantiator, runtimeInfo, featureFlagsProvider);
return new WorkflowActionNode(spec.getName(), spec);
}
use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by caskdata.
the class ApplicationVerificationStage method verifyWorkflowAction.
private void verifyWorkflowAction(ApplicationSpecification appSpec, WorkflowNode node) {
WorkflowActionNode actionNode = (WorkflowActionNode) node;
ScheduleProgramInfo program = actionNode.getProgram();
switch(program.getProgramType()) {
case MAPREDUCE:
Preconditions.checkArgument(appSpec.getMapReduce().containsKey(program.getProgramName()), String.format("MapReduce program '%s' is not configured with the Application.", program.getProgramName()));
break;
case SPARK:
Preconditions.checkArgument(appSpec.getSpark().containsKey(program.getProgramName()), String.format("Spark program '%s' is not configured with the Application.", program.getProgramName()));
break;
case CUSTOM_ACTION:
// no-op
break;
default:
throw new RuntimeException(String.format("Unknown Program '%s' in the Workflow.", program.getProgramName()));
}
}
Aggregations