Search in sources :

Example 1 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project java-dataproc by googleapis.

the class InstantiateInlineWorkflowTemplate method instantiateInlineWorkflowTemplate.

public static void instantiateInlineWorkflowTemplate(String projectId, String region) throws IOException, InterruptedException {
    String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
    // Configure the settings for the workflow template service client.
    WorkflowTemplateServiceSettings workflowTemplateServiceSettings = WorkflowTemplateServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
    // closes the client, but this can also be done manually with the .close() method.
    try (WorkflowTemplateServiceClient workflowTemplateServiceClient = WorkflowTemplateServiceClient.create(workflowTemplateServiceSettings)) {
        // Configure the jobs within the workflow.
        HadoopJob teragenHadoopJob = HadoopJob.newBuilder().setMainJarFileUri("file:///usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar").addArgs("teragen").addArgs("1000").addArgs("hdfs:///gen/").build();
        OrderedJob teragen = OrderedJob.newBuilder().setHadoopJob(teragenHadoopJob).setStepId("teragen").build();
        HadoopJob terasortHadoopJob = HadoopJob.newBuilder().setMainJarFileUri("file:///usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar").addArgs("terasort").addArgs("hdfs:///gen/").addArgs("hdfs:///sort/").build();
        OrderedJob terasort = OrderedJob.newBuilder().setHadoopJob(terasortHadoopJob).addPrerequisiteStepIds("teragen").setStepId("terasort").build();
        // Configure the cluster placement for the workflow.
        // Leave "ZoneUri" empty for "Auto Zone Placement".
        // GceClusterConfig gceClusterConfig =
        // GceClusterConfig.newBuilder().setZoneUri("").build();
        GceClusterConfig gceClusterConfig = GceClusterConfig.newBuilder().setZoneUri("us-central1-a").build();
        ClusterConfig clusterConfig = ClusterConfig.newBuilder().setGceClusterConfig(gceClusterConfig).build();
        ManagedCluster managedCluster = ManagedCluster.newBuilder().setClusterName("my-managed-cluster").setConfig(clusterConfig).build();
        WorkflowTemplatePlacement workflowTemplatePlacement = WorkflowTemplatePlacement.newBuilder().setManagedCluster(managedCluster).build();
        // Create the inline workflow template.
        WorkflowTemplate workflowTemplate = WorkflowTemplate.newBuilder().addJobs(teragen).addJobs(terasort).setPlacement(workflowTemplatePlacement).build();
        // Submit the instantiated inline workflow template request.
        String parent = RegionName.format(projectId, region);
        OperationFuture<Empty, WorkflowMetadata> instantiateInlineWorkflowTemplateAsync = workflowTemplateServiceClient.instantiateInlineWorkflowTemplateAsync(parent, workflowTemplate);
        instantiateInlineWorkflowTemplateAsync.get();
        // Print out a success message.
        System.out.printf("Workflow ran successfully.");
    } catch (ExecutionException e) {
        System.err.println(String.format("Error running workflow: %s ", e.getMessage()));
    }
}
Also used : GceClusterConfig(com.google.cloud.dataproc.v1.GceClusterConfig) WorkflowMetadata(com.google.cloud.dataproc.v1.WorkflowMetadata) HadoopJob(com.google.cloud.dataproc.v1.HadoopJob) WorkflowTemplateServiceClient(com.google.cloud.dataproc.v1.WorkflowTemplateServiceClient) OrderedJob(com.google.cloud.dataproc.v1.OrderedJob) WorkflowTemplate(com.google.cloud.dataproc.v1.WorkflowTemplate) Empty(com.google.protobuf.Empty) ManagedCluster(com.google.cloud.dataproc.v1.ManagedCluster) WorkflowTemplatePlacement(com.google.cloud.dataproc.v1.WorkflowTemplatePlacement) WorkflowTemplateServiceSettings(com.google.cloud.dataproc.v1.WorkflowTemplateServiceSettings) ExecutionException(java.util.concurrent.ExecutionException) ClusterConfig(com.google.cloud.dataproc.v1.ClusterConfig) GceClusterConfig(com.google.cloud.dataproc.v1.GceClusterConfig)

Example 2 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class ProjectClosureTest method testCollectSubWorkflows.

@Test
public void testCollectSubWorkflows() {
    TypedInterface emptyInterface = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
    WorkflowMetadata emptyMetadata = WorkflowMetadata.builder().build();
    PartialWorkflowIdentifier rewrittenSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    WorkflowIdentifier subWorkflowRef = WorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    WorkflowIdentifier otherSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
    PartialWorkflowIdentifier rewrittenNestedSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("nested").version("version").domain("domain").build();
    WorkflowIdentifier nestedSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("nested").version("version").domain("domain").build();
    PartialWorkflowIdentifier rewrittenNestedOtherSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("nested-other").version("version").domain("domain").build();
    WorkflowIdentifier nestedOtherSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("nested-other").version("version").domain("domain").build();
    WorkflowNode workflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenSubWorkflowRef)).build();
    WorkflowNode nestedWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenNestedSubWorkflowRef)).build();
    WorkflowNode nestedOtherWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenNestedOtherSubWorkflowRef)).build();
    WorkflowTemplate emptyWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of()).outputs(ImmutableList.of()).build();
    WorkflowTemplate nestedWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("nested-node").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(nestedOtherWorkflowNode).build())).outputs(ImmutableList.of()).build();
    Operand opTrue = Operand.ofPrimitive(Primitive.ofBooleanValue(true));
    BooleanExpression exprTrue = BooleanExpression.ofComparison(ComparisonExpression.builder().leftValue(opTrue).rightValue(opTrue).operator(ComparisonExpression.Operator.EQ).build());
    List<Node> nodes = ImmutableList.of(Node.builder().id("node-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build(), // Same sub-workflow
    Node.builder().id("node-2").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build(), // Sub-workflow which has a nested sub-workflow in branch (nestedOtherWorkflowNode)
    Node.builder().id("node-3").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).branchNode(BranchNode.builder().ifElse(IfElseBlock.builder().case_(IfBlock.builder().condition(exprTrue).thenNode(Node.builder().id("node-4").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(nestedWorkflowNode).build()).build()).other(ImmutableList.of()).build()).build()).build());
    // nestedOtherWorkflowNode is not in the previous list because
    // that node belongs to the template of a sub-workflow
    Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows = ImmutableMap.of(subWorkflowRef, emptyWorkflowTemplate, otherSubWorkflowRef, emptyWorkflowTemplate, nestedSubWorkflowRef, nestedWorkflowTemplate, nestedOtherSubWorkflowRef, emptyWorkflowTemplate);
    Map<WorkflowIdentifier, WorkflowTemplate> collectedSubWorkflows = ProjectClosure.collectSubWorkflows(nodes, allWorkflows);
    assertThat(collectedSubWorkflows, equalTo(ImmutableMap.of(subWorkflowRef, emptyWorkflowTemplate, nestedSubWorkflowRef, nestedWorkflowTemplate, nestedOtherSubWorkflowRef, emptyWorkflowTemplate)));
}
Also used : TypedInterface(org.flyte.api.v1.TypedInterface) WorkflowMetadata(org.flyte.api.v1.WorkflowMetadata) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) BooleanExpression(org.flyte.api.v1.BooleanExpression) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) Operand(org.flyte.api.v1.Operand) BranchNode(org.flyte.api.v1.BranchNode) WorkflowNode(org.flyte.api.v1.WorkflowNode) Node(org.flyte.api.v1.Node) WorkflowNode(org.flyte.api.v1.WorkflowNode) Test(org.junit.jupiter.api.Test)

Example 3 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class ProtoUtilTest method shouldSerializeWorkflowTemplate.

@Test
void shouldSerializeWorkflowTemplate() {
    Node nodeA = createNode("a").toBuilder().upstreamNodeIds(singletonList("b")).build();
    Node nodeB = createNode("b").toBuilder().metadata(NodeMetadata.builder().name("fancy-b").timeout(Duration.ofMinutes(15)).retries(RetryStrategy.builder().retries(3).build()).build()).build();
    ;
    WorkflowMetadata metadata = WorkflowMetadata.builder().build();
    TypedInterface interface_ = TypedInterface.builder().inputs(emptyMap()).outputs(emptyMap()).build();
    WorkflowTemplate template = WorkflowTemplate.builder().nodes(asList(nodeA, nodeB)).metadata(metadata).interface_(interface_).outputs(emptyList()).build();
    Workflow.Node expectedNode1 = Workflow.Node.newBuilder().setId("a").addUpstreamNodeIds("b").setTaskNode(Workflow.TaskNode.newBuilder().setReferenceId(IdentifierOuterClass.Identifier.newBuilder().setResourceType(TASK).setDomain(DOMAIN).setProject(PROJECT).setName("task-a").setVersion("version-a").build()).build()).addInputs(Literals.Binding.newBuilder().setVar("input-name-a").setBinding(Literals.BindingData.newBuilder().setScalar(Literals.Scalar.newBuilder().setPrimitive(Literals.Primitive.newBuilder().setStringValue("input-scalar-a").build()).build()).build()).build()).build();
    Workflow.Node expectedNode2 = Workflow.Node.newBuilder().setId("b").setMetadata(Workflow.NodeMetadata.newBuilder().setName("fancy-b").setTimeout(com.google.protobuf.Duration.newBuilder().setSeconds(15 * 60).build()).setRetries(Literals.RetryStrategy.newBuilder().setRetries(3).build()).build()).setTaskNode(Workflow.TaskNode.newBuilder().setReferenceId(IdentifierOuterClass.Identifier.newBuilder().setResourceType(TASK).setDomain(DOMAIN).setProject(PROJECT).setName("task-b").setVersion("version-b").build()).build()).addInputs(Literals.Binding.newBuilder().setVar("input-name-b").setBinding(Literals.BindingData.newBuilder().setScalar(Literals.Scalar.newBuilder().setPrimitive(Literals.Primitive.newBuilder().setStringValue("input-scalar-b").build()).build()).build()).build()).build();
    Workflow.WorkflowTemplate serializedTemplate = ProtoUtil.serialize(template);
    assertThat(serializedTemplate, equalTo(Workflow.WorkflowTemplate.newBuilder().setMetadata(Workflow.WorkflowMetadata.newBuilder().build()).setInterface(Interface.TypedInterface.newBuilder().setOutputs(Interface.VariableMap.newBuilder().build()).setInputs(Interface.VariableMap.newBuilder().build()).build()).addNodes(expectedNode1).addNodes(expectedNode2).build()));
}
Also used : WorkflowMetadata(org.flyte.api.v1.WorkflowMetadata) TypedInterface(org.flyte.api.v1.TypedInterface) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) TaskNode(org.flyte.api.v1.TaskNode) WorkflowNode(org.flyte.api.v1.WorkflowNode) BranchNode(org.flyte.api.v1.BranchNode) Node(org.flyte.api.v1.Node) Workflow(flyteidl.core.Workflow) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 4 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class ProjectClosure method checkCycles.

static boolean checkCycles(WorkflowIdentifier workflowId, Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows, Set<WorkflowIdentifier> beingVisited, Set<WorkflowIdentifier> visited) {
    beingVisited.add(workflowId);
    WorkflowTemplate workflow = allWorkflows.get(workflowId);
    List<Node> nodes = workflow.nodes().stream().flatMap(ProjectClosure::flatBranch).collect(toUnmodifiableList());
    for (Node node : nodes) {
        if (isSubWorkflowNode(node)) {
            PartialWorkflowIdentifier partialSubWorkflowId = Objects.requireNonNull(node.workflowNode()).reference().subWorkflowRef();
            WorkflowIdentifier subWorkflowId = WorkflowIdentifier.builder().project(partialSubWorkflowId.project()).name(partialSubWorkflowId.name()).domain(partialSubWorkflowId.domain()).version(partialSubWorkflowId.version()).build();
            if (// backward edge
            beingVisited.contains(subWorkflowId) || (!visited.contains(subWorkflowId) && checkCycles(subWorkflowId, allWorkflows, beingVisited, visited))) {
                return true;
            }
        }
    }
    beingVisited.remove(workflowId);
    visited.add(workflowId);
    return false;
}
Also used : PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) WorkflowNode(org.flyte.api.v1.WorkflowNode) Node(org.flyte.api.v1.Node)

Example 5 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class SdkWorkflowBuilderTest method testConditionalWorkflowIdl.

@Test
void testConditionalWorkflowIdl() {
    SdkWorkflowBuilder builder = new SdkWorkflowBuilder();
    new ConditionalWorkflow().expand(builder);
    Node caseNode = Node.builder().id("neq").taskNode(TaskNode.builder().referenceId(PartialTaskIdentifier.builder().name("org.flyte.flytekit.SdkWorkflowBuilderTest$MultiplicationTask").build()).build()).inputs(asList(Binding.builder().var_("a").binding(BindingData.ofOutputReference(OutputReference.builder().var("in").nodeId(Node.START_NODE_ID).build())).build(), Binding.builder().var_("b").binding(BindingData.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(2L)))).build())).upstreamNodeIds(emptyList()).build();
    IfElseBlock ifElse = IfElseBlock.builder().case_(IfBlock.builder().condition(BooleanExpression.ofComparison(ComparisonExpression.builder().leftValue(Operand.ofVar("$0")).rightValue(Operand.ofPrimitive(Primitive.ofIntegerValue(2L))).operator(ComparisonExpression.Operator.NEQ).build())).thenNode(caseNode).build()).error(NodeError.builder().message("No cases matched").failedNodeId("square").build()).other(emptyList()).build();
    Node expectedNode = Node.builder().id("square").branchNode(BranchNode.builder().ifElse(ifElse).build()).inputs(singletonList(Binding.builder().var_("$0").binding(BindingData.ofOutputReference(OutputReference.builder().var("in").nodeId(Node.START_NODE_ID).build())).build())).upstreamNodeIds(emptyList()).build();
    WorkflowTemplate expected = WorkflowTemplate.builder().metadata(WorkflowMetadata.builder().build()).interface_(expectedInterface()).outputs(expectedOutputs()).nodes(singletonList(expectedNode)).build();
    assertEquals(expected, builder.toIdlTemplate());
}
Also used : IfElseBlock(org.flyte.api.v1.IfElseBlock) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) TaskNode(org.flyte.api.v1.TaskNode) BranchNode(org.flyte.api.v1.BranchNode) Node(org.flyte.api.v1.Node) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

WorkflowTemplate (org.flyte.api.v1.WorkflowTemplate)18 Test (org.junit.jupiter.api.Test)12 Node (org.flyte.api.v1.Node)9 Literal (org.flyte.api.v1.Literal)7 RunnableTask (org.flyte.api.v1.RunnableTask)7 WorkflowIdentifier (org.flyte.api.v1.WorkflowIdentifier)6 PartialWorkflowIdentifier (org.flyte.api.v1.PartialWorkflowIdentifier)5 TaskNode (org.flyte.api.v1.TaskNode)5 WorkflowNode (org.flyte.api.v1.WorkflowNode)5 BranchNode (org.flyte.api.v1.BranchNode)4 TypedInterface (org.flyte.api.v1.TypedInterface)4 WorkflowMetadata (org.flyte.api.v1.WorkflowMetadata)4 List (java.util.List)3 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)3 WorkflowTemplate (com.google.cloud.dataproc.v1.WorkflowTemplate)2 HashMap (java.util.HashMap)2 DynamicWorkflowTask (org.flyte.api.v1.DynamicWorkflowTask)2 RetryableTask (org.flyte.localengine.examples.RetryableTask)2 RetryableWorkflow (org.flyte.localengine.examples.RetryableWorkflow)2 Test (org.junit.Test)2