Examples with RunnableTask org.flyte.api.v1.RunnableTask used on opensource projects

Example 11 with RunnableTask

use of org.flyte.api.v1.RunnableTask in project flytekit-java by flyteorg.

the class LocalEngineTest method testFibonacci.

@Test
void testFibonacci() {
    String workflowName = new FibonacciWorkflow().getName();
    Literal fib0 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(0L)));
    Literal fib1 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(1L)));
    Literal fib2 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(1L)));
    Literal fib3 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(2L)));
    Literal fib4 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(3L)));
    Literal fib5 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(5L)));
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    TestingListener listener = new TestingListener();
    Map<String, Literal> outputs = LocalEngine.compileAndExecute(workflows.get(workflowName), tasks, emptyMap(), ImmutableMap.of("fib0", fib0, "fib1", fib1), listener);
    assertEquals(ImmutableMap.of("fib4", fib4, "fib5", fib5), outputs);
    assertEquals(ImmutableList.<List<Object>>builder().add(ofPending("fib-2")).add(ofPending("fib-3")).add(ofPending("fib-4")).add(ofPending("fib-5")).add(ofStarting("fib-2", ImmutableMap.of("a", fib0, "b", fib1))).add(ofCompleted("fib-2", ImmutableMap.of("a", fib0, "b", fib1), ImmutableMap.of("c", fib2))).add(ofStarting("fib-3", ImmutableMap.of("a", fib1, "b", fib2))).add(ofCompleted("fib-3", ImmutableMap.of("a", fib1, "b", fib2), ImmutableMap.of("c", fib3))).add(ofStarting("fib-4", ImmutableMap.of("a", fib2, "b", fib3))).add(ofCompleted("fib-4", ImmutableMap.of("a", fib2, "b", fib3), ImmutableMap.of("c", fib4))).add(ofStarting("fib-5", ImmutableMap.of("a", fib3, "b", fib4))).add(ofCompleted("fib-5", ImmutableMap.of("a", fib3, "b", fib4), ImmutableMap.of("c", fib5))).build(), listener.actions);
}

Example 12 with RunnableTask

use of org.flyte.api.v1.RunnableTask in project flytekit-java by flyteorg.

the class LocalEngineTest method testRetryableTask_completed.

@Test
public void testRetryableTask_completed() {
    String workflowName = new RetryableWorkflow().getName();
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    WorkflowTemplate workflow = workflows.get(workflowName);
    TestingListener listener = new TestingListener();
    // make sure we don't run two tests in parallel
    synchronized (RetryableTask.class) {
        RetryableTask.ATTEMPTS_BEFORE_SUCCESS.set(5L);
        RetryableTask.ATTEMPTS.set(0L);
        LocalEngine.compileAndExecute(workflow, tasks, emptyMap(), ImmutableMap.of(), listener);
        assertEquals(ImmutableList.<List<Object>>builder().add(ofPending("node-1")).add(ofStarting("node-1", ImmutableMap.of())).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        1)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        2)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        3)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        4)).add(ofCompleted("node-1", ImmutableMap.of(), ImmutableMap.of())).build(), listener.actions);
        // will finish on attempt number 5 according to ATTEMPTS_BEFORE_SUCCESS
        assertEquals(5L, RetryableTask.ATTEMPTS.get());
    }
}

Example 13 with RunnableTask

use of org.flyte.api.v1.RunnableTask in project flytekit-java by flyteorg.

the class LocalEngineTest method testBindingCollection.

@Test
public void testBindingCollection() {
    String workflowName = new ListWorkflow().getName();
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    WorkflowTemplate workflow = workflows.get(workflowName);
    Map<String, Literal> outputs = LocalEngine.compileAndExecute(workflow, tasks, emptyMap(), ImmutableMap.of());
    // 3 = 1 + 2, 7 = 3 + 4
    Literal i3 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(3)));
    Literal i7 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(7)));
    assertEquals(ImmutableMap.of("list", Literal.ofCollection(ImmutableList.of(i3, i7))), outputs);
}

Example 14 with RunnableTask

use of org.flyte.api.v1.RunnableTask in project flytekit-java by flyteorg.

the class SdkRunnableTaskRegistrarTest method shouldLoadRunnableTasksFromDiscoveredRegistries.

@Test
void shouldLoadRunnableTasksFromDiscoveredRegistries() {
    // given
    String testTaskName = "org.flyte.flytekit.SdkRunnableTaskRegistrarTest$TestTask";
    String otherTestTaskName = "org.flyte.flytekit.SdkRunnableTaskRegistrarTest$OtherTestTask";
    TaskIdentifier expectedTestTaskId = TaskIdentifier.builder().project("project").domain("domain").name(testTaskName).version("version").build();
    TypedInterface typedInterface = TypedInterface.builder().inputs(SdkTypes.nulls().getVariableMap()).outputs(SdkTypes.nulls().getVariableMap()).build();
    RetryStrategy retries = RetryStrategy.builder().retries(0).build();
    RetryStrategy otherRetries = RetryStrategy.builder().retries(1).build();
    Map<Resources.ResourceName, String> limits = new HashMap<>();
    limits.put(Resources.ResourceName.CPU, "0.5");
    limits.put(Resources.ResourceName.MEMORY, "2Gi");
    Map<Resources.ResourceName, String> requests = new HashMap<>();
    requests.put(Resources.ResourceName.CPU, "2");
    requests.put(Resources.ResourceName.MEMORY, "5Gi");
    Resources resources = Resources.builder().limits(limits).requests(requests).build();
    RunnableTask expectedTask = createRunnableTask(testTaskName, typedInterface, retries, null);
    TaskIdentifier expectedOtherTestTaskId = TaskIdentifier.builder().project("project").domain("domain").name(otherTestTaskName).version("version").build();
    RunnableTask expectedOtherTask = createRunnableTask(otherTestTaskName, typedInterface, otherRetries, resources);
    // when
    Map<TaskIdentifier, RunnableTask> tasks = registrar.load(ENV);
    // then
    assertAll(() -> assertThat(tasks, hasKey(is(expectedTestTaskId))), () -> assertThat(tasks, hasKey(is(expectedOtherTestTaskId))));
    assertTaskEquals(tasks.get(expectedTestTaskId), expectedTask);
    assertTaskEquals(tasks.get(expectedOtherTestTaskId), expectedOtherTask);
}

Example 15 with RunnableTask

use of org.flyte.api.v1.RunnableTask in project flytekit-java by flyteorg.

the class ExecutionNodeCompiler method compile.

static ExecutionNode compile(Node node, Map<String, RunnableTask> runnableTasks, Map<String, DynamicWorkflowTask> dynamicWorkflowTasks) {
    List<String> upstreamNodeIds = new ArrayList<>();
    node.inputs().stream().map(Binding::binding).flatMap(ExecutionNodeCompiler::unpackBindingData).filter(x -> x.kind() == BindingData.Kind.PROMISE).map(x -> x.promise().nodeId()).forEach(upstreamNodeIds::add);
    upstreamNodeIds.addAll(node.upstreamNodeIds());
    if (upstreamNodeIds.isEmpty()) {
        upstreamNodeIds.add(START_NODE_ID);
    }
    if (node.branchNode() != null) {
        throw new IllegalArgumentException("BranchNode isn't yet supported for local execution");
    }
    if (node.workflowNode() != null) {
        throw new IllegalArgumentException("WorkflowNode isn't yet supported for local execution");
    }
    String taskName = node.taskNode().referenceId().name();
    DynamicWorkflowTask dynamicWorkflowTask = dynamicWorkflowTasks.get(taskName);
    RunnableTask runnableTask = runnableTasks.get(taskName);
    if (dynamicWorkflowTask != null) {
        throw new IllegalArgumentException("DynamicWorkflowTask isn't yet supported for local execution");
    }
    Objects.requireNonNull(runnableTask, () -> String.format("Couldn't find task [%s]", taskName));
    int attempts = runnableTask.getRetries().retries() + 1;
    return ExecutionNode.builder().nodeId(node.id()).bindings(node.inputs()).runnableTask(runnableTask).upstreamNodeIds(upstreamNodeIds).attempts(attempts).build();
}