Examples with WorkflowTemplate org.flyte.api.v1.WorkflowTemplate used on opensource projects

Example 11 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class ExecuteDynamicWorkflow method execute.

private void execute() {
    Config config = Config.load();
    ExecutionConfig executionConfig = ExecutionConfig.load();
    Collection<ClassLoader> modules = ClassLoaders.forModuleDir(config.moduleDir()).values();
    Map<String, FileSystem> fileSystems = FileSystemLoader.loadFileSystems(modules);
    FileSystem outputFs = FileSystemLoader.getFileSystem(fileSystems, outputPrefix);
    ProtoWriter protoWriter = new ProtoWriter(outputPrefix, outputFs);
    try {
        FileSystem inputFs = FileSystemLoader.getFileSystem(fileSystems, inputs);
        ProtoReader protoReader = new ProtoReader(inputFs);
        TaskTemplate taskTemplate = protoReader.getTaskTemplate(taskTemplatePath);
        ClassLoader packageClassLoader = PackageLoader.load(fileSystems, taskTemplate);
        Map<String, String> env = getEnv();
        Map<WorkflowIdentifier, WorkflowTemplate> workflowTemplates = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(WorkflowTemplateRegistrar.class, env));
        Map<TaskIdentifier, RunnableTask> runnableTasks = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(RunnableTaskRegistrar.class, env));
        Map<TaskIdentifier, DynamicWorkflowTask> dynamicWorkflowTasks = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(DynamicWorkflowTaskRegistrar.class, env));
        // before we run anything, switch class loader, otherwise,
        // ServiceLoaders and other things wouldn't work, for instance,
        // FileSystemRegister in Apache Beam
        // we don't take the whole "custom" field, but only jflyte part, for that we ser-de it
        Struct custom = JFlyteCustom.deserializeFromStruct(taskTemplate.custom()).serializeToStruct();
        // all tasks already have staged jars, we can reuse 'jflyte' custom from current task to get
        // it
        Map<TaskIdentifier, TaskTemplate> taskTemplates = mapValues(ProjectClosure.createTaskTemplates(executionConfig, runnableTasks, dynamicWorkflowTasks), template -> template.toBuilder().custom(ProjectClosure.merge(template.custom(), custom)).build());
        DynamicJobSpec futures = withClassLoader(packageClassLoader, () -> {
            Map<String, Literal> input = protoReader.getInput(inputs);
            DynamicWorkflowTask task = getDynamicWorkflowTask(this.task);
            return task.run(input);
        });
        DynamicJobSpec rewrittenFutures = rewrite(executionConfig, futures, taskTemplates, workflowTemplates);
        if (rewrittenFutures.nodes().isEmpty()) {
            Map<String, Literal> outputs = getLiteralMap(rewrittenFutures.outputs());
            protoWriter.writeOutputs(outputs);
        } else {
            protoWriter.writeFutures(rewrittenFutures);
        }
    } catch (ContainerError e) {
        LOG.error("failed to run dynamic workflow", e);
        protoWriter.writeError(ProtoUtil.serializeContainerError(e));
    } catch (Throwable e) {
        LOG.error("failed to run dynamic workflow", e);
        protoWriter.writeError(ProtoUtil.serializeThrowable(e));
    }
}

Example 12 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class ExecuteLocal method call.

@Override
public Integer call() {
    Map<String, ClassLoader> modules = ExecuteLocalLoader.loadModules(packageDir);
    Map<String, String> env = ImmutableMap.of("FLYTE_INTERNAL_DOMAIN", "development", "FLYTE_INTERNAL_VERSION", "test", "FLYTE_INTERNAL_PROJECT", "flytetester");
    Map<String, RunnableTask> runnableTasks = ExecuteLocalLoader.loadTasks(modules, env);
    Map<String, DynamicWorkflowTask> dynamicWorkflowTasks = // TODO support dynamic tasks
    emptyMap();
    Map<String, WorkflowTemplate> workflows = ExecuteLocalLoader.loadWorkflows(modules, env);
    WorkflowTemplate workflow = Preconditions.checkNotNull(workflows.get(workflowName), "workflow not found [%s]", workflowName);
    String synopsis = getCustomSynopsis();
    List<String> inputArgsList = inputArgs == null ? Collections.emptyList() : Arrays.asList(inputArgs);
    Map<String, Literal> inputs = getArgsParser().parseInputs(synopsis, workflow.interface_().inputs(), inputArgsList);
    try {
        // TODO, use logging listener here
        ExecutionListener listener = NoopExecutionListener.create();
        Map<String, Literal> outputs = LocalEngine.compileAndExecute(workflow, runnableTasks, dynamicWorkflowTasks, inputs, listener);
        LOG.info("Outputs: " + StringUtil.serializeLiteralMap(outputs));
        return 0;
    } catch (Throwable e) {
        return handleException(e);
    }
}

Example 13 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class ProjectClosureTest method testCheckCycles.

@Test
public void testCheckCycles() {
    TypedInterface emptyInterface = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
    WorkflowMetadata emptyMetadata = WorkflowMetadata.builder().build();
    PartialWorkflowIdentifier rewrittenWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    PartialWorkflowIdentifier otherRewrittenWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
    WorkflowIdentifier workflowRef = WorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    WorkflowIdentifier otherWorkflowRef = WorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
    WorkflowNode workflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenWorkflowRef)).build();
    WorkflowNode otherWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(otherRewrittenWorkflowRef)).build();
    WorkflowTemplate workflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("sub-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(otherWorkflowNode).build())).outputs(ImmutableList.of()).build();
    WorkflowTemplate otherWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("sub-2").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build())).outputs(ImmutableList.of()).build();
    Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows = ImmutableMap.of(workflowRef, workflowTemplate, otherWorkflowRef, otherWorkflowTemplate);
    // workflow -> otherWorkflow -> workflow
    IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> ProjectClosure.checkCycles(allWorkflows));
    assertEquals("Workflow [WorkflowIdentifier{name=name, domain=domain, project=project, version=version}] " + "cannot have itself as a node, directly or indirectly", exception.getMessage());
}

Example 14 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class LocalEngineTest method testRetryableTask_failed.

@Test
public void testRetryableTask_failed() {
    String workflowName = new RetryableWorkflow().getName();
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    WorkflowTemplate workflow = workflows.get(workflowName);
    TestingListener listener = new TestingListener();
    // make sure we don't run two tests in parallel
    synchronized (RetryableTask.class) {
        // will never succeed within retry limit
        RetryableTask.ATTEMPTS_BEFORE_SUCCESS.set(10);
        RetryableTask.ATTEMPTS.set(0L);
        RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> LocalEngine.compileAndExecute(workflow, tasks, emptyMap(), ImmutableMap.of(), listener));
        assertEquals("oops", e.getMessage());
        assertEquals(ImmutableList.<List<Object>>builder().add(ofPending("node-1")).add(ofStarting("node-1", ImmutableMap.of())).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        1)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        2)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        3)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        4)).add(ofRetrying("node-1", ImmutableMap.of(), "oops", /* attempt= */
        5)).add(ofError("node-1", ImmutableMap.of(), "oops")).build(), listener.actions);
        // getRetries() returns 5, so we have 6 attempts/executions total
        assertEquals(6L, RetryableTask.ATTEMPTS.get());
    }
}

Example 15 with WorkflowTemplate

use of org.flyte.api.v1.WorkflowTemplate in project flytekit-java by flyteorg.

the class LocalEngineTest method testBindingMap.

@Test
public void testBindingMap() {
    String workflowName = new MapWorkflow().getName();
    Map<String, WorkflowTemplate> workflows = loadWorkflows();
    Map<String, RunnableTask> tasks = loadTasks();
    WorkflowTemplate workflow = workflows.get(workflowName);
    Map<String, Literal> outputs = LocalEngine.compileAndExecute(workflow, tasks, emptyMap(), ImmutableMap.of());
    // 3 = 1 + 2, 7 = 3 + 4
    Literal i3 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(3)));
    Literal i7 = Literal.ofScalar(Scalar.ofPrimitive(Primitive.ofIntegerValue(7)));
    assertEquals(ImmutableMap.of("map", Literal.ofMap(ImmutableMap.of("e", i3, "f", i7))), outputs);
}