Search in sources :

Example 26 with Node

use of com.marcnuri.yakc.model.io.k8s.api.core.v1.Node in project flytekit-java by flyteorg.

the class ProjectClosureTest method testCollectSubWorkflows.

@Test
public void testCollectSubWorkflows() {
    TypedInterface emptyInterface = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
    WorkflowMetadata emptyMetadata = WorkflowMetadata.builder().build();
    PartialWorkflowIdentifier rewrittenSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    WorkflowIdentifier subWorkflowRef = WorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    WorkflowIdentifier otherSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
    PartialWorkflowIdentifier rewrittenNestedSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("nested").version("version").domain("domain").build();
    WorkflowIdentifier nestedSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("nested").version("version").domain("domain").build();
    PartialWorkflowIdentifier rewrittenNestedOtherSubWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("nested-other").version("version").domain("domain").build();
    WorkflowIdentifier nestedOtherSubWorkflowRef = WorkflowIdentifier.builder().project("project").name("nested-other").version("version").domain("domain").build();
    WorkflowNode workflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenSubWorkflowRef)).build();
    WorkflowNode nestedWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenNestedSubWorkflowRef)).build();
    WorkflowNode nestedOtherWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenNestedOtherSubWorkflowRef)).build();
    WorkflowTemplate emptyWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of()).outputs(ImmutableList.of()).build();
    WorkflowTemplate nestedWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("nested-node").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(nestedOtherWorkflowNode).build())).outputs(ImmutableList.of()).build();
    Operand opTrue = Operand.ofPrimitive(Primitive.ofBooleanValue(true));
    BooleanExpression exprTrue = BooleanExpression.ofComparison(ComparisonExpression.builder().leftValue(opTrue).rightValue(opTrue).operator(ComparisonExpression.Operator.EQ).build());
    List<Node> nodes = ImmutableList.of(Node.builder().id("node-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build(), // Same sub-workflow
    Node.builder().id("node-2").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build(), // Sub-workflow which has a nested sub-workflow in branch (nestedOtherWorkflowNode)
    Node.builder().id("node-3").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).branchNode(BranchNode.builder().ifElse(IfElseBlock.builder().case_(IfBlock.builder().condition(exprTrue).thenNode(Node.builder().id("node-4").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(nestedWorkflowNode).build()).build()).other(ImmutableList.of()).build()).build()).build());
    // nestedOtherWorkflowNode is not in the previous list because
    // that node belongs to the template of a sub-workflow
    Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows = ImmutableMap.of(subWorkflowRef, emptyWorkflowTemplate, otherSubWorkflowRef, emptyWorkflowTemplate, nestedSubWorkflowRef, nestedWorkflowTemplate, nestedOtherSubWorkflowRef, emptyWorkflowTemplate);
    Map<WorkflowIdentifier, WorkflowTemplate> collectedSubWorkflows = ProjectClosure.collectSubWorkflows(nodes, allWorkflows);
    assertThat(collectedSubWorkflows, equalTo(ImmutableMap.of(subWorkflowRef, emptyWorkflowTemplate, nestedSubWorkflowRef, nestedWorkflowTemplate, nestedOtherSubWorkflowRef, emptyWorkflowTemplate)));
}
Also used : TypedInterface(org.flyte.api.v1.TypedInterface) WorkflowMetadata(org.flyte.api.v1.WorkflowMetadata) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) BooleanExpression(org.flyte.api.v1.BooleanExpression) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) Operand(org.flyte.api.v1.Operand) BranchNode(org.flyte.api.v1.BranchNode) WorkflowNode(org.flyte.api.v1.WorkflowNode) Node(org.flyte.api.v1.Node) WorkflowNode(org.flyte.api.v1.WorkflowNode) Test(org.junit.jupiter.api.Test)

Example 27 with Node

use of com.marcnuri.yakc.model.io.k8s.api.core.v1.Node in project flytekit-java by flyteorg.

the class ProtoUtilTest method shouldRejectNonDNS1123NodesIds.

@ParameterizedTest
@ValueSource(strings = { "id with spaces is invalid", "CamelCaseIsInvalid" })
void shouldRejectNonDNS1123NodesIds(String nodeId) {
    Node node = Node.builder().id(nodeId).upstreamNodeIds(emptyList()).inputs(emptyList()).build();
    IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> ProtoUtil.serialize(node));
    assertEquals("Node id [" + nodeId + "] must conform to DNS 1123 naming format", ex.getMessage());
}
Also used : TaskNode(org.flyte.api.v1.TaskNode) WorkflowNode(org.flyte.api.v1.WorkflowNode) BranchNode(org.flyte.api.v1.BranchNode) Node(org.flyte.api.v1.Node) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 28 with Node

use of com.marcnuri.yakc.model.io.k8s.api.core.v1.Node in project flytekit-java by flyteorg.

the class ProtoUtilTest method shouldSerializeWorkflowTemplate.

@Test
void shouldSerializeWorkflowTemplate() {
    Node nodeA = createNode("a").toBuilder().upstreamNodeIds(singletonList("b")).build();
    Node nodeB = createNode("b").toBuilder().metadata(NodeMetadata.builder().name("fancy-b").timeout(Duration.ofMinutes(15)).retries(RetryStrategy.builder().retries(3).build()).build()).build();
    ;
    WorkflowMetadata metadata = WorkflowMetadata.builder().build();
    TypedInterface interface_ = TypedInterface.builder().inputs(emptyMap()).outputs(emptyMap()).build();
    WorkflowTemplate template = WorkflowTemplate.builder().nodes(asList(nodeA, nodeB)).metadata(metadata).interface_(interface_).outputs(emptyList()).build();
    Workflow.Node expectedNode1 = Workflow.Node.newBuilder().setId("a").addUpstreamNodeIds("b").setTaskNode(Workflow.TaskNode.newBuilder().setReferenceId(IdentifierOuterClass.Identifier.newBuilder().setResourceType(TASK).setDomain(DOMAIN).setProject(PROJECT).setName("task-a").setVersion("version-a").build()).build()).addInputs(Literals.Binding.newBuilder().setVar("input-name-a").setBinding(Literals.BindingData.newBuilder().setScalar(Literals.Scalar.newBuilder().setPrimitive(Literals.Primitive.newBuilder().setStringValue("input-scalar-a").build()).build()).build()).build()).build();
    Workflow.Node expectedNode2 = Workflow.Node.newBuilder().setId("b").setMetadata(Workflow.NodeMetadata.newBuilder().setName("fancy-b").setTimeout(com.google.protobuf.Duration.newBuilder().setSeconds(15 * 60).build()).setRetries(Literals.RetryStrategy.newBuilder().setRetries(3).build()).build()).setTaskNode(Workflow.TaskNode.newBuilder().setReferenceId(IdentifierOuterClass.Identifier.newBuilder().setResourceType(TASK).setDomain(DOMAIN).setProject(PROJECT).setName("task-b").setVersion("version-b").build()).build()).addInputs(Literals.Binding.newBuilder().setVar("input-name-b").setBinding(Literals.BindingData.newBuilder().setScalar(Literals.Scalar.newBuilder().setPrimitive(Literals.Primitive.newBuilder().setStringValue("input-scalar-b").build()).build()).build()).build()).build();
    Workflow.WorkflowTemplate serializedTemplate = ProtoUtil.serialize(template);
    assertThat(serializedTemplate, equalTo(Workflow.WorkflowTemplate.newBuilder().setMetadata(Workflow.WorkflowMetadata.newBuilder().build()).setInterface(Interface.TypedInterface.newBuilder().setOutputs(Interface.VariableMap.newBuilder().build()).setInputs(Interface.VariableMap.newBuilder().build()).build()).addNodes(expectedNode1).addNodes(expectedNode2).build()));
}
Also used : WorkflowMetadata(org.flyte.api.v1.WorkflowMetadata) TypedInterface(org.flyte.api.v1.TypedInterface) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) TaskNode(org.flyte.api.v1.TaskNode) WorkflowNode(org.flyte.api.v1.WorkflowNode) BranchNode(org.flyte.api.v1.BranchNode) Node(org.flyte.api.v1.Node) Workflow(flyteidl.core.Workflow) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 29 with Node

use of com.marcnuri.yakc.model.io.k8s.api.core.v1.Node in project flytekit-java by flyteorg.

the class ProjectClosure method checkCycles.

static boolean checkCycles(WorkflowIdentifier workflowId, Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows, Set<WorkflowIdentifier> beingVisited, Set<WorkflowIdentifier> visited) {
    beingVisited.add(workflowId);
    WorkflowTemplate workflow = allWorkflows.get(workflowId);
    List<Node> nodes = workflow.nodes().stream().flatMap(ProjectClosure::flatBranch).collect(toUnmodifiableList());
    for (Node node : nodes) {
        if (isSubWorkflowNode(node)) {
            PartialWorkflowIdentifier partialSubWorkflowId = Objects.requireNonNull(node.workflowNode()).reference().subWorkflowRef();
            WorkflowIdentifier subWorkflowId = WorkflowIdentifier.builder().project(partialSubWorkflowId.project()).name(partialSubWorkflowId.name()).domain(partialSubWorkflowId.domain()).version(partialSubWorkflowId.version()).build();
            if (// backward edge
            beingVisited.contains(subWorkflowId) || (!visited.contains(subWorkflowId) && checkCycles(subWorkflowId, allWorkflows, beingVisited, visited))) {
                return true;
            }
        }
    }
    beingVisited.remove(workflowId);
    visited.add(workflowId);
    return false;
}
Also used : PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) WorkflowNode(org.flyte.api.v1.WorkflowNode) Node(org.flyte.api.v1.Node)

Example 30 with Node

use of com.marcnuri.yakc.model.io.k8s.api.core.v1.Node in project flytekit-java by flyteorg.

the class IdentifierRewriteTest method shouldRewriteBranchNodes.

@Test
void shouldRewriteBranchNodes() {
    ComparisonExpression comparison = ComparisonExpression.builder().operator(ComparisonExpression.Operator.EQ).leftValue(Operand.ofVar("a")).rightValue(Operand.ofVar("b")).build();
    BooleanExpression condition = BooleanExpression.ofComparison(comparison);
    PartialTaskIdentifier partialReference = PartialTaskIdentifier.builder().name("task-name").build();
    PartialTaskIdentifier rewrittenReference = PartialTaskIdentifier.builder().name("task-name").domain("rewritten-domain").version("rewritten-version").project("rewritten-project").build();
    TaskNode partialTaskNode = TaskNode.builder().referenceId(partialReference).build();
    TaskNode rewrittenTaskNode = TaskNode.builder().referenceId(rewrittenReference).build();
    Node partialNode = Node.builder().id("node-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).taskNode(partialTaskNode).build();
    Node rewrittenNode = Node.builder().id("node-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).taskNode(rewrittenTaskNode).build();
    IfBlock partialIfBlock = IfBlock.builder().condition(condition).thenNode(partialNode).build();
    IfBlock rewrittenIfBlock = IfBlock.builder().condition(condition).thenNode(rewrittenNode).build();
    BranchNode partialBranchNode = BranchNode.builder().ifElse(IfElseBlock.builder().case_(partialIfBlock).other(ImmutableList.of(partialIfBlock)).elseNode(partialNode).build()).build();
    BranchNode rewrittenBranchNode = BranchNode.builder().ifElse(IfElseBlock.builder().case_(rewrittenIfBlock).other(ImmutableList.of(rewrittenIfBlock)).elseNode(rewrittenNode).build()).build();
    assertThat(rewriter.visitor().visitBranchNode(partialBranchNode), equalTo(rewrittenBranchNode));
}
Also used : ComparisonExpression(org.flyte.api.v1.ComparisonExpression) BooleanExpression(org.flyte.api.v1.BooleanExpression) TaskNode(org.flyte.api.v1.TaskNode) BranchNode(org.flyte.api.v1.BranchNode) TaskNode(org.flyte.api.v1.TaskNode) BranchNode(org.flyte.api.v1.BranchNode) WorkflowNode(org.flyte.api.v1.WorkflowNode) Node(org.flyte.api.v1.Node) PartialTaskIdentifier(org.flyte.api.v1.PartialTaskIdentifier) IfBlock(org.flyte.api.v1.IfBlock) Test(org.junit.jupiter.api.Test)

Aggregations

Node (org.openstreetmap.osmosis.core.domain.v0_6.Node)52 Test (org.junit.Test)27 CommonEntityData (org.openstreetmap.osmosis.core.domain.v0_6.CommonEntityData)26 WayNode (org.openstreetmap.osmosis.core.domain.v0_6.WayNode)21 Date (java.util.Date)18 NodeContainer (org.openstreetmap.osmosis.core.container.v0_6.NodeContainer)17 Tag (org.openstreetmap.osmosis.core.domain.v0_6.Tag)16 Node (org.flyte.api.v1.Node)15 Test (org.junit.jupiter.api.Test)15 ArrayList (java.util.ArrayList)14 OsmUser (org.openstreetmap.osmosis.core.domain.v0_6.OsmUser)14 Way (org.openstreetmap.osmosis.core.domain.v0_6.Way)13 TaskNode (org.flyte.api.v1.TaskNode)10 Bound (org.openstreetmap.osmosis.core.domain.v0_6.Bound)10 List (java.util.List)9 Map (java.util.Map)9 Node (org.eclipse.smarthome.binding.mqtt.generic.internal.convention.homie300.Node)9 Node (org.openhab.binding.mqtt.homie.internal.homie300.Node)9 EntityContainer (org.openstreetmap.osmosis.core.container.v0_6.EntityContainer)9 Node (ch.ethz.globis.phtree.v16.Node)8