use of org.flyte.api.v1.WorkflowTemplate in project java-dataproc by googleapis.
the class InstantiateInlineWorkflowTemplate method instantiateInlineWorkflowTemplate.
public static void instantiateInlineWorkflowTemplate(String projectId, String region) throws IOException, InterruptedException {
String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);
// Configure the settings for the workflow template service client.
WorkflowTemplateServiceSettings workflowTemplateServiceSettings = WorkflowTemplateServiceSettings.newBuilder().setEndpoint(myEndpoint).build();
// closes the client, but this can also be done manually with the .close() method.
try (WorkflowTemplateServiceClient workflowTemplateServiceClient = WorkflowTemplateServiceClient.create(workflowTemplateServiceSettings)) {
// Configure the jobs within the workflow.
HadoopJob teragenHadoopJob = HadoopJob.newBuilder().setMainJarFileUri("file:///usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar").addArgs("teragen").addArgs("1000").addArgs("hdfs:///gen/").build();
OrderedJob teragen = OrderedJob.newBuilder().setHadoopJob(teragenHadoopJob).setStepId("teragen").build();
HadoopJob terasortHadoopJob = HadoopJob.newBuilder().setMainJarFileUri("file:///usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar").addArgs("terasort").addArgs("hdfs:///gen/").addArgs("hdfs:///sort/").build();
OrderedJob terasort = OrderedJob.newBuilder().setHadoopJob(terasortHadoopJob).addPrerequisiteStepIds("teragen").setStepId("terasort").build();
// Configure the cluster placement for the workflow.
// Leave "ZoneUri" empty for "Auto Zone Placement".
// GceClusterConfig gceClusterConfig =
// GceClusterConfig.newBuilder().setZoneUri("").build();
GceClusterConfig gceClusterConfig = GceClusterConfig.newBuilder().setZoneUri("us-central1-a").build();
ClusterConfig clusterConfig = ClusterConfig.newBuilder().setGceClusterConfig(gceClusterConfig).build();
ManagedCluster managedCluster = ManagedCluster.newBuilder().setClusterName("my-managed-cluster").setConfig(clusterConfig).build();
WorkflowTemplatePlacement workflowTemplatePlacement = WorkflowTemplatePlacement.newBuilder().setManagedCluster(managedCluster).build();
// Create the inline workflow template.
WorkflowTemplate workflowTemplate = WorkflowTemplate.newBuilder().addJobs(teragen).addJobs(terasort).setPlacement(workflowTemplatePlacement).build();
// Submit the instantiated inline workflow template request.
String parent = RegionName.format(projectId, region);
OperationFuture<Empty, WorkflowMetadata> instantiateInlineWorkflowTemplateAsync = workflowTemplateServiceClient.instantiateInlineWorkflowTemplateAsync(parent, workflowTemplate);
instantiateInlineWorkflowTemplateAsync.get();
// Print out a success message.
System.out.printf("Workflow ran successfully.");
} catch (ExecutionException e) {
System.err.println(String.format("Error running workflow: %s ", e.getMessage()));
}
}
use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.
the class ProjectClosureTest method testCollectSubWorkflows.
@Test
public void testCollectSubWorkflows() {
TypedInterface emptyInterface = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
WorkflowMetadata emptyMetadata = WorkflowMetadata.builder().build();
PartialWorkflowIdentifier rewrittenSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
WorkflowIdentifier subWorkflowRef = WorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
WorkflowIdentifier otherSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
PartialWorkflowIdentifier rewrittenNestedSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("nested").version("version").domain("domain").build();
WorkflowIdentifier nestedSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("nested").version("version").domain("domain").build();
PartialWorkflowIdentifier rewrittenNestedOtherSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("nested-other").version("version").domain("domain").build();
WorkflowIdentifier nestedOtherSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("nested-other").version("version").domain("domain").build();
WorkflowNode workflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenSubWorkflowRef)).build();
WorkflowNode nestedWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenNestedSubWorkflowRef)).build();
WorkflowNode nestedOtherWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenNestedOtherSubWorkflowRef)).build();
WorkflowTemplate emptyWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of()).outputs(ImmutableList.of()).build();
WorkflowTemplate nestedWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("nested-node").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(nestedOtherWorkflowNode).build())).outputs(ImmutableList.of()).build();
Operand opTrue = Operand.ofPrimitive(Primitive.ofBooleanValue(true));
BooleanExpression exprTrue = BooleanExpression.ofComparison(ComparisonExpression.builder().leftValue(opTrue).rightValue(opTrue).operator(ComparisonExpression.Operator.EQ).build());
List<Node> nodes = ImmutableList.of(Node.builder().id("node-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build(), // Same sub-workflow
Node.builder().id("node-2").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build(), // Sub-workflow which has a nested sub-workflow in branch (nestedOtherWorkflowNode)
Node.builder().id("node-3").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).branchNode(BranchNode.builder().ifElse(IfElseBlock.builder().case_(IfBlock.builder().condition(exprTrue).thenNode(Node.builder().id("node-4").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(nestedWorkflowNode).build()).build()).other(ImmutableList.of()).build()).build()).build());
// nestedOtherWorkflowNode is not in the previous list because
// that node belongs to the template of a sub-workflow
Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows = ImmutableMap.of(subWorkflowRef, emptyWorkflowTemplate, otherSubWorkflowRef, emptyWorkflowTemplate, nestedSubWorkflowRef, nestedWorkflowTemplate, nestedOtherSubWorkflowRef, emptyWorkflowTemplate);
Map<WorkflowIdentifier, WorkflowTemplate> collectedSubWorkflows = ProjectClosure.collectSubWorkflows(nodes, allWorkflows);
assertThat(collectedSubWorkflows, equalTo(ImmutableMap.of(subWorkflowRef, emptyWorkflowTemplate, nestedSubWorkflowRef, nestedWorkflowTemplate, nestedOtherSubWorkflowRef, emptyWorkflowTemplate)));
}
use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.
the class ProtoUtilTest method shouldSerializeWorkflowTemplate.
@Test
void shouldSerializeWorkflowTemplate() {
Node nodeA = createNode("a").toBuilder().upstreamNodeIds(singletonList("b")).build();
Node nodeB = createNode("b").toBuilder().metadata(NodeMetadata.builder().name("fancy-b").timeout(Duration.ofMinutes(15)).retries(RetryStrategy.builder().retries(3).build()).build()).build();
;
WorkflowMetadata metadata = WorkflowMetadata.builder().build();
TypedInterface interface_ = TypedInterface.builder().inputs(emptyMap()).outputs(emptyMap()).build();
WorkflowTemplate template = WorkflowTemplate.builder().nodes(asList(nodeA, nodeB)).metadata(metadata).interface_(interface_).outputs(emptyList()).build();
Workflow.Node expectedNode1 = Workflow.Node.newBuilder().setId("a").addUpstreamNodeIds("b").setTaskNode(Workflow.TaskNode.newBuilder().setReferenceId(IdentifierOuterClass.Identifier.newBuilder().setResourceType(TASK).setDomain(DOMAIN).setProject(PROJECT).setName("task-a").setVersion("version-a").build()).build()).addInputs(Literals.Binding.newBuilder().setVar("input-name-a").setBinding(Literals.BindingData.newBuilder().setScalar(Literals.Scalar.newBuilder().setPrimitive(Literals.Primitive.newBuilder().setStringValue("input-scalar-a").build()).build()).build()).build()).build();
Workflow.Node expectedNode2 = Workflow.Node.newBuilder().setId("b").setMetadata(Workflow.NodeMetadata.newBuilder().setName("fancy-b").setTimeout(com.google.protobuf.Duration.newBuilder().setSeconds(15 * 60).build()).setRetries(Literals.RetryStrategy.newBuilder().setRetries(3).build()).build()).setTaskNode(Workflow.TaskNode.newBuilder().setReferenceId(IdentifierOuterClass.Identifier.newBuilder().setResourceType(TASK).setDomain(DOMAIN).setProject(PROJECT).setName("task-b").setVersion("version-b").build()).build()).addInputs(Literals.Binding.newBuilder().setVar("input-name-b").setBinding(Literals.BindingData.newBuilder().setScalar(Literals.Scalar.newBuilder().setPrimitive(Literals.Primitive.newBuilder().setStringValue("input-scalar-b").build()).build()).build()).build()).build();
Workflow.WorkflowTemplate serializedTemplate = ProtoUtil.serialize(template);
assertThat(serializedTemplate, equalTo(Workflow.WorkflowTemplate.newBuilder().setMetadata(Workflow.WorkflowMetadata.newBuilder().build()).setInterface(Interface.TypedInterface.newBuilder().setOutputs(Interface.VariableMap.newBuilder().build()).setInputs(Interface.VariableMap.newBuilder().build()).build()).addNodes(expectedNode1).addNodes(expectedNode2).build()));
}
use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.
the class ProjectClosure method checkCycles.
static boolean checkCycles(WorkflowIdentifier workflowId, Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows, Set<WorkflowIdentifier> beingVisited, Set<WorkflowIdentifier> visited) {
beingVisited.add(workflowId);
WorkflowTemplate workflow = allWorkflows.get(workflowId);
List<Node> nodes = workflow.nodes().stream().flatMap(ProjectClosure::flatBranch).collect(toUnmodifiableList());
for (Node node : nodes) {
if (isSubWorkflowNode(node)) {
PartialWorkflowIdentifier partialSubWorkflowId = Objects.requireNonNull(node.workflowNode()).reference().subWorkflowRef();
WorkflowIdentifier subWorkflowId = WorkflowIdentifier.builder().project(partialSubWorkflowId.project()).name(partialSubWorkflowId.name()).domain(partialSubWorkflowId.domain()).version(partialSubWorkflowId.version()).build();
if (// backward edge
beingVisited.contains(subWorkflowId) || (!visited.contains(subWorkflowId) && checkCycles(subWorkflowId, allWorkflows, beingVisited, visited))) {
return true;
}
}
}
beingVisited.remove(workflowId);
visited.add(workflowId);
return false;
}
use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.
the class SdkWorkflowBuilderTest method testConditionalWorkflowIdl.
@Test
void testConditionalWorkflowIdl() {
SdkWorkflowBuilder builder = new SdkWorkflowBuilder();
new ConditionalWorkflow().expand(builder);
Node caseNode = Node.builder().id("neq").taskNode(TaskNode.builder().referenceId(PartialTaskIdentifier.builder().name("org.flyte.flytekit.SdkWorkflowBuilderTest$MultiplicationTask").build()).build()).inputs(asList(Binding.builder().var_("a").binding(BindingData.ofOutputReference(OutputReference.builder().var("in").nodeId(Node.START_NODE_ID).build())).build(), Binding.builder().var_("b").binding(BindingData.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(2L)))).build())).upstreamNodeIds(emptyList()).build();
IfElseBlock ifElse = IfElseBlock.builder().case_(IfBlock.builder().condition(BooleanExpression.ofComparison(ComparisonExpression.builder().leftValue(Operand.ofVar("$0")).rightValue(Operand.ofPrimitive(Primitive.ofIntegerValue(2L))).operator(ComparisonExpression.Operator.NEQ).build())).thenNode(caseNode).build()).error(NodeError.builder().message("No cases matched").failedNodeId("square").build()).other(emptyList()).build();
Node expectedNode = Node.builder().id("square").branchNode(BranchNode.builder().ifElse(ifElse).build()).inputs(singletonList(Binding.builder().var_("$0").binding(BindingData.ofOutputReference(OutputReference.builder().var("in").nodeId(Node.START_NODE_ID).build())).build())).upstreamNodeIds(emptyList()).build();
WorkflowTemplate expected = WorkflowTemplate.builder().metadata(WorkflowMetadata.builder().build()).interface_(expectedInterface()).outputs(expectedOutputs()).nodes(singletonList(expectedNode)).build();
assertEquals(expected, builder.toIdlTemplate());
}
Aggregations