Examples with WorkflowNode co.cask.cdap.api.workflow.WorkflowNode used on opensource projects

Example 1 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class WorkflowSpecificationCodec method deserialize.

@Override
public WorkflowSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();
    String className = jsonObj.get("className").getAsString();
    String name = jsonObj.get("name").getAsString();
    String description = jsonObj.get("description").getAsString();
    Map<String, String> properties = deserializeMap(jsonObj.get("properties"), context, String.class);
    List<WorkflowNode> nodes = deserializeList(jsonObj.get("nodes"), context, WorkflowNode.class);
    Map<String, DatasetCreationSpec> localDatasetSpec = deserializeMap(jsonObj.get("localDatasetSpecs"), context, DatasetCreationSpec.class);
    return new WorkflowSpecification(className, name, description, properties, nodes, localDatasetSpec);
}

Example 2 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class ProgramSystemMetadataWriter method getWorkflowNodes.

private Iterable<String> getWorkflowNodes() {
    if (ProgramType.WORKFLOW != programId.getType()) {
        return ImmutableSet.of();
    }
    Preconditions.checkArgument(programSpec instanceof WorkflowSpecification, "Expected programSpec %s to be of type WorkflowSpecification", programSpec);
    WorkflowSpecification workflowSpec = (WorkflowSpecification) this.programSpec;
    Set<String> workflowNodeNames = new HashSet<>();
    for (Map.Entry<String, WorkflowNode> entry : workflowSpec.getNodeIdMap().entrySet()) {
        WorkflowNode workflowNode = entry.getValue();
        WorkflowNodeType type = workflowNode.getType();
        // Fork nodes have integers as node ids. Ignore them in system metadata.
        if (WorkflowNodeType.FORK == type) {
            continue;
        }
        workflowNodeNames.add(entry.getKey());
    }
    return workflowNodeNames;
}

Example 3 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class ApplicationVerificationStage method verifyWorkflowNodeList.

private void verifyWorkflowNodeList(ApplicationSpecification appSpec, WorkflowSpecification workflowSpec, List<WorkflowNode> nodeList, Set<String> existingNodeNames) {
    for (WorkflowNode n : nodeList) {
        if (existingNodeNames.contains(n.getNodeId())) {
            throw new RuntimeException(String.format("Node '%s' already exists in workflow '%s'.", n.getNodeId(), workflowSpec.getName()));
        }
        existingNodeNames.add(n.getNodeId());
        verifyWorkflowNode(appSpec, workflowSpec, n, existingNodeNames);
    }
}

Example 4 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class WorkflowVerificationTest method verifyWorkflowWithLocalDatasetSpecification.

private void verifyWorkflowWithLocalDatasetSpecification(ApplicationSpecification appSpec) {
    WorkflowSpecification spec = appSpec.getWorkflows().get("WorkflowWithLocalDatasets");
    List<WorkflowNode> nodes = spec.getNodes();
    Assert.assertTrue(nodes.size() == 2);
    WorkflowNode node = nodes.get(0);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    WorkflowActionNode actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
    node = nodes.get(1);
    Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
    actionNode = (WorkflowActionNode) node;
    Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
    Map<String, DatasetCreationSpec> localDatasetSpecs = spec.getLocalDatasetSpecs();
    Assert.assertEquals(5, localDatasetSpecs.size());
    DatasetCreationSpec datasetCreationSpec = localDatasetSpecs.get("mytable");
    Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfile");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfile_with_properties");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals("prop_value", datasetCreationSpec.getProperties().getProperties().get("prop_key"));
    datasetCreationSpec = localDatasetSpecs.get("mytablefromtype");
    Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
    datasetCreationSpec = localDatasetSpecs.get("myfilefromtype");
    Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
    Assert.assertEquals("another_prop_value", datasetCreationSpec.getProperties().getProperties().get("another_prop_key"));
    // Check if the application specification has correct modules
    Map<String, String> datasetModules = appSpec.getDatasetModules();
    Assert.assertEquals(2, datasetModules.size());
    Assert.assertTrue(datasetModules.containsKey(FileSet.class.getName()));
    Assert.assertTrue(datasetModules.containsKey(Table.class.getName()));
}

Example 5 with WorkflowNode

use of co.cask.cdap.api.workflow.WorkflowNode in project cdap by caskdata.

the class WorkflowDriver method executeAll.

private void executeAll(Iterator<WorkflowNode> iterator, ApplicationSpecification appSpec, InstantiatorFactory instantiator, ClassLoader classLoader, WorkflowToken token) {
    while (iterator.hasNext() && runningThread != null) {
        try {
            blockIfSuspended();
            WorkflowNode node = iterator.next();
            executeNode(appSpec, node, instantiator, classLoader, token);
        } catch (Throwable t) {
            Throwable rootCause = Throwables.getRootCause(t);
            if (rootCause instanceof InterruptedException) {
                LOG.debug("Workflow '{}' with run id '{}' aborted", workflowSpec.getName(), workflowRunId.getRun());
                basicWorkflowContext.setState(new ProgramState(ProgramStatus.KILLED, rootCause.getMessage()));
                break;
            }
            basicWorkflowContext.setState(new ProgramState(ProgramStatus.FAILED, rootCause.getMessage()));
            throw Throwables.propagate(rootCause);
        }
    }
}