Examples with WorkflowNode co.cask.cdap.api.workflow.WorkflowNode used on opensource projects

Example 11 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class DistributedWorkflowProgramRunner method findDriverResources.

/**
 * Returns the {@link Resources} requirement for the workflow runnable deduced by Spark
 * or MapReduce driver resources requirement.
 */
private Resources findDriverResources(Map<String, SparkSpecification> sparkSpecs, Map<String, MapReduceSpecification> mrSpecs, WorkflowSpecification spec) {
    // Find the resource requirements from the workflow with 768MB as minimum.
    // It is the largest memory and cores from all Spark and MapReduce programs inside the workflow
    Resources resources = new Resources(768);
    for (WorkflowNode node : spec.getNodeIdMap().values()) {
        if (WorkflowNodeType.ACTION == node.getType()) {
            ScheduleProgramInfo programInfo = ((WorkflowActionNode) node).getProgram();
            SchedulableProgramType programType = programInfo.getProgramType();
            if (programType == SchedulableProgramType.SPARK || programType == SchedulableProgramType.MAPREDUCE) {
                // The program spec shouldn't be null, otherwise the Workflow is not valid
                Resources driverResources;
                if (programType == SchedulableProgramType.SPARK) {
                    driverResources = sparkSpecs.get(programInfo.getProgramName()).getClientResources();
                } else {
                    driverResources = mrSpecs.get(programInfo.getProgramName()).getDriverResources();
                }
                if (driverResources != null) {
                    resources = max(resources, driverResources);
                }
            }
        }
    }
    return resources;
}

Example 12 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class WorkflowDriver method executeFork.

private void executeFork(final ApplicationSpecification appSpec, WorkflowForkNode fork, final InstantiatorFactory instantiator, final ClassLoader classLoader, final WorkflowToken token) throws Exception {
    CountDownLatch executorTerminateLatch = new CountDownLatch(1);
    ExecutorService executorService = createExecutor(fork.getBranches().size(), executorTerminateLatch, "fork-" + fork.getNodeId() + "-%d");
    CompletionService<Map.Entry<String, WorkflowToken>> completionService = new ExecutorCompletionService<>(executorService);
    try {
        for (final List<WorkflowNode> branch : fork.getBranches()) {
            completionService.submit(new Callable<Map.Entry<String, WorkflowToken>>() {

                @Override
                public Map.Entry<String, WorkflowToken> call() throws Exception {
                    WorkflowToken copiedToken = ((BasicWorkflowToken) token).deepCopy();
                    executeAll(branch.iterator(), appSpec, instantiator, classLoader, copiedToken);
                    return Maps.immutableEntry(branch.toString(), copiedToken);
                }
            });
        }
        for (int i = 0; i < fork.getBranches().size(); i++) {
            try {
                Future<Map.Entry<String, WorkflowToken>> forkBranchResult = completionService.take();
                Map.Entry<String, WorkflowToken> retValue = forkBranchResult.get();
                String branchInfo = retValue.getKey();
                WorkflowToken branchToken = retValue.getValue();
                ((BasicWorkflowToken) token).mergeToken(branchToken);
                LOG.trace("Execution of branch {} for fork {} completed.", branchInfo, fork);
            } catch (InterruptedException e) {
                // Due to workflow abortion, so just break the loop
                break;
            } catch (ExecutionException e) {
                // Unwrap the cause
                Throwables.propagateIfPossible(e.getCause(), Exception.class);
                throw Throwables.propagate(e.getCause());
            }
        }
    } finally {
        // Update the WorkflowToken after the execution of the FORK node completes.
        runtimeStore.updateWorkflowToken(workflowRunId, token);
        executorService.shutdownNow();
        // Wait for the executor termination
        executorTerminateLatch.await();
    }
}

Example 13 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class DefaultStore method recordCompletedWorkflow.

private void recordCompletedWorkflow(AppMetadataStore metaStore, WorkflowDataset workflowDataset, WorkflowId workflowId, String runId) {
    RunRecordMeta runRecord = metaStore.getRun(workflowId.run(runId));
    if (runRecord == null) {
        return;
    }
    ApplicationId app = workflowId.getParent();
    ApplicationSpecification appSpec = getApplicationSpec(metaStore, app);
    if (appSpec == null || appSpec.getWorkflows() == null || appSpec.getWorkflows().get(workflowId.getProgram()) == null) {
        LOG.warn("Missing ApplicationSpecification for {}, " + "potentially caused by application removal right after stopping workflow {}", app, workflowId);
        return;
    }
    boolean workFlowNodeFailed = false;
    WorkflowSpecification workflowSpec = appSpec.getWorkflows().get(workflowId.getProgram());
    Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap();
    List<WorkflowDataset.ProgramRun> programRunsList = new ArrayList<>();
    for (Map.Entry<String, String> entry : runRecord.getProperties().entrySet()) {
        if (!("workflowToken".equals(entry.getKey()) || "runtimeArgs".equals(entry.getKey()) || "workflowNodeState".equals(entry.getKey()))) {
            WorkflowActionNode workflowNode = (WorkflowActionNode) nodeIdMap.get(entry.getKey());
            ProgramType programType = ProgramType.valueOfSchedulableType(workflowNode.getProgram().getProgramType());
            ProgramId innerProgram = app.program(programType, entry.getKey());
            RunRecordMeta innerProgramRun = metaStore.getRun(innerProgram.run(entry.getValue()));
            if (innerProgramRun != null && innerProgramRun.getStatus().equals(ProgramRunStatus.COMPLETED)) {
                Long stopTs = innerProgramRun.getStopTs();
                // since the program is completed, the stop ts cannot be null
                if (stopTs == null) {
                    LOG.warn("Since the program has completed, expected its stop time to not be null. " + "Not writing workflow completed record for Program = {}, Workflow = {}, Run = {}", innerProgram, workflowId, runRecord);
                    workFlowNodeFailed = true;
                    break;
                }
                programRunsList.add(new WorkflowDataset.ProgramRun(entry.getKey(), entry.getValue(), programType, stopTs - innerProgramRun.getStartTs()));
            } else {
                workFlowNodeFailed = true;
                break;
            }
        }
    }
    if (workFlowNodeFailed) {
        return;
    }
    workflowDataset.write(workflowId, runRecord, programRunsList);
}

Example 14 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class WorkflowVerificationTest method verifyAnotherGoodWorkflowSpecification.

private void verifyAnotherGoodWorkflowSpecification(ApplicationSpecification appSpec) {
    WorkflowSpecification spec = appSpec.getWorkflows().get("AnotherGoodWorkflow");
    List<WorkflowNode> nodes = spec.getNodes();
    Assert.assertTrue(nodes.size() == 4);
    WorkflowNode node = nodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    WorkflowActionNode actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
    node = nodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
    WorkflowForkNode fork = (WorkflowForkNode) node;
    Assert.assertTrue(fork.getBranches().size() == 2);
    List<WorkflowNode> forkBranch1 = fork.getBranches().get(0);
    Assert.assertTrue(forkBranch1.size() == 3);
    node = forkBranch1.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR2")));
    node = forkBranch1.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
    WorkflowConditionNode condition = (WorkflowConditionNode) node;
    Assert.assertTrue(condition.getPredicateClassName().contains("MyVerificationPredicate"));
    List<WorkflowNode> ifNodes = condition.getIfBranch();
    Assert.assertTrue(ifNodes.size() == 2);
    List<WorkflowNode> elseNodes = condition.getElseBranch();
    Assert.assertTrue(elseNodes.size() == 2);
    node = ifNodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR3")));
    node = ifNodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR4")));
    node = elseNodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR5")));
    node = elseNodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR6")));
    node = forkBranch1.get(2);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR7")));
    List<WorkflowNode> forkBranch2 = fork.getBranches().get(1);
    Assert.assertTrue(forkBranch2.size() == 1);
    node = forkBranch2.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR8")));
    node = nodes.get(2);
    Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
    ifNodes = ((WorkflowConditionNode) node).getIfBranch();
    elseNodes = ((WorkflowConditionNode) node).getElseBranch();
    Assert.assertTrue(ifNodes.size() == 2);
    node = ifNodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
    node = ifNodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP2")));
    Assert.assertTrue(elseNodes.size() == 3);
    node = elseNodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP3")));
    node = elseNodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP4")));
    node = elseNodes.get(2);
    Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
    WorkflowForkNode anotherFork = (WorkflowForkNode) node;
    Assert.assertTrue(anotherFork.getBranches().size() == 2);
    List<WorkflowNode> anotherForkBranch1 = anotherFork.getBranches().get(0);
    Assert.assertTrue(anotherForkBranch1.size() == 1);
    List<WorkflowNode> anotherForkBranch2 = anotherFork.getBranches().get(1);
    Assert.assertTrue(anotherForkBranch2.size() == 1);
    node = anotherForkBranch1.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP5")));
    node = anotherForkBranch2.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP6")));
    node = nodes.get(3);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP7")));
}

Example 15 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class WorkflowVerificationTest method verifyGoodWorkflowSpecifications.

private void verifyGoodWorkflowSpecifications(ApplicationSpecification appSpec) {
    WorkflowSpecification spec = appSpec.getWorkflows().get("GoodWorkflow");
    Assert.assertTrue(spec.getNodes().size() == 4);
    List<WorkflowNode> nodes = spec.getNodes();
    WorkflowNode node = nodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    WorkflowActionNode actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    node = nodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
    WorkflowForkNode fork1 = (WorkflowForkNode) node;
    Assert.assertTrue(fork1.getBranches().size() == 2);
    List<WorkflowNode> fork1Branch1 = fork1.getBranches().get(0);
    Assert.assertTrue(fork1Branch1.size() == 2);
    List<WorkflowNode> fork1Branch2 = fork1.getBranches().get(1);
    Assert.assertTrue(fork1Branch2.size() == 1);
    actionNode = (WorkflowActionNode) fork1Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    WorkflowForkNode fork2 = (WorkflowForkNode) fork1Branch1.get(1);
    List<WorkflowNode> fork2Branch1 = fork2.getBranches().get(0);
    Assert.assertTrue(fork2Branch1.size() == 2);
    List<WorkflowNode> fork2Branch2 = fork2.getBranches().get(1);
    Assert.assertTrue(fork2Branch2.size() == 1);
    actionNode = (WorkflowActionNode) fork2Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    WorkflowForkNode fork3 = (WorkflowForkNode) fork2Branch1.get(1);
    List<WorkflowNode> fork3Branch1 = fork3.getBranches().get(0);
    Assert.assertTrue(fork3Branch1.size() == 2);
    List<WorkflowNode> fork3Branch2 = fork3.getBranches().get(1);
    Assert.assertTrue(fork3Branch2.size() == 1);
    WorkflowForkNode fork4 = (WorkflowForkNode) fork3Branch1.get(0);
    List<WorkflowNode> fork4Branch1 = fork4.getBranches().get(0);
    Assert.assertTrue(fork4Branch1.size() == 2);
    List<WorkflowNode> fork4Branch2 = fork4.getBranches().get(1);
    Assert.assertTrue(fork4Branch2.size() == 1);
    actionNode = (WorkflowActionNode) fork4Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    actionNode = (WorkflowActionNode) fork4Branch1.get(1);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork4Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork4Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork4Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork2Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork1Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    node = nodes.get(2);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
    node = nodes.get(3);
    WorkflowForkNode fork5 = (WorkflowForkNode) node;
    List<WorkflowNode> fork5Branch1 = fork5.getBranches().get(0);
    Assert.assertTrue(fork5Branch1.size() == 1);
    List<WorkflowNode> fork5Branch2 = fork5.getBranches().get(1);
    Assert.assertTrue(fork5Branch2.size() == 1);
    actionNode = (WorkflowActionNode) fork5Branch1.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.CUSTOM_ACTION, "DummyAction")));
    Assert.assertNotNull(actionNode.getCustomActionSpecification());
    actionNode = (WorkflowActionNode) fork5Branch2.get(0);
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "DummyMR")));
    Assert.assertNull(actionNode.getCustomActionSpecification());
}