Examples with WorkflowTemplate com.google.cloud.dataproc.v1.WorkflowTemplate used on opensource projects

Example 16 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class ProjectClosureTest method testCheckCycles.

@Test
public void testCheckCycles() {
    TypedInterface emptyInterface = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
    WorkflowMetadata emptyMetadata = WorkflowMetadata.builder().build();
    PartialWorkflowIdentifier rewrittenWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    PartialWorkflowIdentifier otherRewrittenWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
    WorkflowIdentifier workflowRef = WorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    WorkflowIdentifier otherWorkflowRef = WorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
    WorkflowNode workflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenWorkflowRef)).build();
    WorkflowNode otherWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(otherRewrittenWorkflowRef)).build();
    WorkflowTemplate workflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("sub-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(otherWorkflowNode).build())).outputs(ImmutableList.of()).build();
    WorkflowTemplate otherWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("sub-2").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build())).outputs(ImmutableList.of()).build();
    Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows = ImmutableMap.of(workflowRef, workflowTemplate, otherWorkflowRef, otherWorkflowTemplate);
    // workflow -> otherWorkflow -> workflow
    IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> ProjectClosure.checkCycles(allWorkflows));
    assertEquals("Workflow [WorkflowIdentifier{name=name, domain=domain, project=project, version=version}] " + "cannot have itself as a node, directly or indirectly", exception.getMessage());
}

Example 17 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class LocalEngineTest method testRetryableTask_failed.

@Test
public void testRetryableTask_failed() {
    String workflowName = new RetryableWorkflow().getName();
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    WorkflowTemplate workflow = workflows.get(workflowName);
    TestingListener listener = new TestingListener();
    // make sure we don't run two tests in parallel
    synchronized (RetryableTask.class) {
        // will never succeed within retry limit
        RetryableTask.ATTEMPTS_BEFORE_SUCCESS.set(10);
        RetryableTask.ATTEMPTS.set(0L);
        RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> LocalEngine.compileAndExecute(workflow, tasks, emptyMap(), ImmutableMap.of(), listener));
        assertEquals("oops", e.getMessage());
        assertEquals(ImmutableList.<List<Object>>builder().add(ofPending("node-1")).add(ofStarting("node-1", ImmutableMap.of())).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        1)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        2)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        3)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        4)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        5)).add(ofError("node-1", ImmutableMap.of(), "oops")).build(), listener.actions);
        // getRetries() returns 5, so we have 6 attempts/executions total
        assertEquals(6L, RetryableTask.ATTEMPTS.get());
    }
}

Example 18 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class LocalEngineTest method testBindingMap.

@Test
public void testBindingMap() {
    String workflowName = new MapWorkflow().getName();
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    WorkflowTemplate workflow = workflows.get(workflowName);
    Map<String, Literal> outputs = LocalEngine.compileAndExecute(workflow, tasks, emptyMap(), ImmutableMap.of());
    // 3 = 1 + 2, 7 = 3 + 4
    Literal i3 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(3)));
    Literal i7 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(7)));
    assertEquals(ImmutableMap.of("map", Literal.ofMap(ImmutableMap.of("e", i3, "f", i7))), outputs);
}

Example 19 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class LocalEngineTest method testFibonacci.

@Test
void testFibonacci() {
    String workflowName = new FibonacciWorkflow().getName();
    Literal fib0 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(0L)));
    Literal fib1 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(1L)));
    Literal fib2 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(1L)));
    Literal fib3 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(2L)));
    Literal fib4 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(3L)));
    Literal fib5 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(5L)));
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    TestingListener listener = new TestingListener();
    Map<String, Literal> outputs = LocalEngine.compileAndExecute(workflows.get(workflowName), tasks, emptyMap(), ImmutableMap.of("fib0", fib0, "fib1", fib1), listener);
    assertEquals(ImmutableMap.of("fib4", fib4, "fib5", fib5), outputs);
    assertEquals(ImmutableList.<List<Object>>builder().add(ofPending("fib-2")).add(ofPending("fib-3")).add(ofPending("fib-4")).add(ofPending("fib-5")).add(ofStarting("fib-2", ImmutableMap.of("a", fib0, "b", fib1))).add(ofCompleted("fib-2", ImmutableMap.of("a", fib0, "b", fib1), ImmutableMap.of("c", fib2))).add(ofStarting("fib-3", ImmutableMap.of("a", fib1, "b", fib2))).add(ofCompleted("fib-3", ImmutableMap.of("a", fib1, "b", fib2), ImmutableMap.of("c", fib3))).add(ofStarting("fib-4", ImmutableMap.of("a", fib2, "b", fib3))).add(ofCompleted("fib-4", ImmutableMap.of("a", fib2, "b", fib3), ImmutableMap.of("c", fib4))).add(ofStarting("fib-5", ImmutableMap.of("a", fib3, "b", fib4))).add(ofCompleted("fib-5", ImmutableMap.of("a", fib3, "b", fib4), ImmutableMap.of("c", fib5))).build(), listener.actions);
}

Example 20 with WorkflowTemplate

use of com.google.cloud.dataproc.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class LocalEngineTest method testRetryableTask_completed.

@Test
public void testRetryableTask_completed() {
    String workflowName = new RetryableWorkflow().getName();
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    WorkflowTemplate workflow = workflows.get(workflowName);
    TestingListener listener = new TestingListener();
    // make sure we don't run two tests in parallel
    synchronized (RetryableTask.class) {
        RetryableTask.ATTEMPTS_BEFORE_SUCCESS.set(5L);
        RetryableTask.ATTEMPTS.set(0L);
        LocalEngine.compileAndExecute(workflow, tasks, emptyMap(), ImmutableMap.of(), listener);
        assertEquals(ImmutableList.<List<Object>>builder().add(ofPending("node-1")).add(ofStarting("node-1", ImmutableMap.of())).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        1)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        2)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        3)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        4)).add(ofCompleted("node-1", ImmutableMap.of(), ImmutableMap.of())).build(), listener.actions);
        // will finish on attempt number 5 according to ATTEMPTS_BEFORE_SUCCESS
        assertEquals(5L, RetryableTask.ATTEMPTS.get());
    }
}