Examples with WorkflowTemplate com.google.cloud.dataproc.v1.WorkflowTemplate used on opensource projects

Example 6 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class ProjectClosure method checkCycles.

static boolean checkCycles(WorkflowIdentifier workflowId, Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows, Set<WorkflowIdentifier> beingVisited, Set<WorkflowIdentifier> visited) {
    beingVisited.add(workflowId);
    WorkflowTemplate workflow = allWorkflows.get(workflowId);
    List<Node> nodes = workflow.nodes().stream().flatMap(ProjectClosure::flatBranch).collect(toUnmodifiableList());
    for (Node node : nodes) {
        if (isSubWorkflowNode(node)) {
            PartialWorkflowIdentifier partialSubWorkflowId = Objects.requireNonNull(node.workflowNode()).reference().subWorkflowRef();
            WorkflowIdentifier subWorkflowId = WorkflowIdentifier.builder().project(partialSubWorkflowId.project()).name(partialSubWorkflowId.name()).domain(partialSubWorkflowId.domain()).version(partialSubWorkflowId.version()).build();
            if (// backward edge
            beingVisited.contains(subWorkflowId) || (!visited.contains(subWorkflowId) && checkCycles(subWorkflowId, allWorkflows, beingVisited, visited))) {
                return true;
            }
        }
    }
    beingVisited.remove(workflowId);
    visited.add(workflowId);
    return false;
}

Example 7 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class SdkWorkflowBuilderTest method testConditionalWorkflowIdl.

@Test
void testConditionalWorkflowIdl() {
    SdkWorkflowBuilder builder = new SdkWorkflowBuilder();
    new ConditionalWorkflow().expand(builder);
    Node caseNode = Node.builder().id("neq").taskNode(TaskNode.builder().referenceId(PartialTaskIdentifier.builder().name("org.flyte.flytekit.SdkWorkflowBuilderTest$MultiplicationTask").build()).build()).inputs(asList(Binding.builder().var_("a").binding(BindingData.ofOutputReference(OutputReference.builder().var("in").nodeId(Node.START_NODE_ID).build())).build(), Binding.builder().var_("b").binding(BindingData.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(2L)))).build())).upstreamNodeIds(emptyList()).build();
    IfElseBlock ifElse = IfElseBlock.builder().case_(IfBlock.builder().condition(BooleanExpression.ofComparison(ComparisonExpression.builder().leftValue(Operand.ofVar("$0")).rightValue(Operand.ofPrimitive(Primitive.ofIntegerValue(2L))).operator(ComparisonExpression.Operator.NEQ).build())).thenNode(caseNode).build()).error(NodeError.builder().message("No cases matched").failedNodeId("square").build()).other(emptyList()).build();
    Node expectedNode = Node.builder().id("square").branchNode(BranchNode.builder().ifElse(ifElse).build()).inputs(singletonList(Binding.builder().var_("$0").binding(BindingData.ofOutputReference(OutputReference.builder().var("in").nodeId(Node.START_NODE_ID).build())).build())).upstreamNodeIds(emptyList()).build();
    WorkflowTemplate expected = WorkflowTemplate.builder().metadata(WorkflowMetadata.builder().build()).interface_(expectedInterface()).outputs(expectedOutputs()).nodes(singletonList(expectedNode)).build();
    assertEquals(expected, builder.toIdlTemplate());
}

Example 8 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class LocalEngineTest method testStructWorkflow.

@Test
public void testStructWorkflow() {
    String workflowName = new StructWorkflow().getName();
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    WorkflowTemplate workflow = workflows.get(workflowName);
    Literal inputStructLiteral = Literal.ofScalar(Scalar.ofGeneric(Struct.of(ImmutableMap.of("someKey1", Struct.Value.ofStringValue("some_value_1"), "someKey2", Struct.Value.ofBoolValue(true)))));
    Map<String, Literal> inputs = ImmutableMap.of("someString", Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofStringValue("some_string_value"))), "someStruct", inputStructLiteral);
    Map<String, Literal> outputs = LocalEngine.compileAndExecute(workflow, tasks, emptyMap(), inputs);
    Literal expectedOutput = Literal.ofScalar(Scalar.ofGeneric(Struct.of(ImmutableMap.of("someKey1", Struct.Value.ofStringValue("some_value_1-output"), "someKey2", Struct.Value.ofBoolValue(true)))));
    assertEquals(expectedOutput, outputs.get("outputStructData"));
}

Example 9 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class SdkTestingExecutor method execute.

public Result execute() {
    TestingSdkWorkflowBuilder builder = new TestingSdkWorkflowBuilder(fixedInputMap(), fixedInputTypeMap());
    workflow().expand(builder);
    WorkflowTemplate workflowTemplate = builder.toIdlTemplate();
    for (Node node : workflowTemplate.nodes()) {
        TaskNode taskNode = node.taskNode();
        if (taskNode != null) {
            String taskName = taskNode.referenceId().name();
            checkArgument(fixedTaskMap().containsKey(taskName), "Can't execute remote task [%s], " + "use SdkTestingExecutor#withTaskOutput or SdkTestingExecutor#withTask", taskName);
        }
    }
    Map<String, Literal> outputLiteralMap = LocalEngine.compileAndExecute(workflowTemplate, unmodifiableMap(fixedTaskMap()), emptyMap(), fixedInputMap());
    Map<String, LiteralType> outputLiteralTypeMap = workflowTemplate.interface_().outputs().entrySet().stream().collect(toMap(Map.Entry::getKey, x -> x.getValue().literalType()));
    return Result.create(outputLiteralMap, outputLiteralTypeMap);
}

Example 10 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class SdkWorkflowWithSdkRemoteLaunchPlanTest method applyShouldReturnASdkWorkflowNode.

@Test
void applyShouldReturnASdkWorkflowNode() {
    SdkWorkflowBuilder builder = new SdkWorkflowBuilder();
    new WorkflowExample().expand(builder);
    Node expectedNode = Node.builder().id("some-node-id").workflowNode(WorkflowNode.builder().reference(WorkflowNode.Reference.ofLaunchPlanRef(PartialLaunchPlanIdentifier.builder().domain("dev").project("project-a").name("SomeLaunchPlan").version("version").build())).build()).upstreamNodeIds(Collections.emptyList()).inputs(Arrays.asList(Binding.builder().var_("a").binding(BindingData.ofOutputReference(OutputReference.builder().nodeId("start-node").var("a").build())).build(), Binding.builder().var_("b").binding(BindingData.ofOutputReference(OutputReference.builder().nodeId("start-node").var("b").build())).build())).build();
    WorkflowTemplate expected = WorkflowTemplate.builder().metadata(WorkflowMetadata.builder().build()).interface_(expectedInterface()).outputs(expectedOutputs()).nodes(singletonList(expectedNode)).build();
    assertEquals(expected, builder.toIdlTemplate());
}