Search in sources :

Example 6 with WorkflowIdentifier

use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.

the class FlyteAdminClientTest method shouldPropagateCreateWorkflowToStub.

@Test
public void shouldPropagateCreateWorkflowToStub() {
    String nodeId = "node";
    WorkflowIdentifier identifier = WorkflowIdentifier.builder().domain(DOMAIN).project(PROJECT).name(WF_NAME).version(WF_VERSION).build();
    TaskNode taskNode = TaskNode.builder().referenceId(PartialTaskIdentifier.builder().domain(DOMAIN).project(PROJECT).name(TASK_NAME).version(TASK_VERSION).build()).build();
    Node node = Node.builder().id(nodeId).taskNode(taskNode).inputs(ImmutableList.of(Binding.builder().var_(VAR_NAME).binding(BindingData.ofScalar(Scalar.ofPrimitive(Primitive.ofStringValue(SCALAR)))).build())).upstreamNodeIds(emptyList()).build();
    TypedInterface interface_ = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
    WorkflowTemplate template = WorkflowTemplate.builder().nodes(ImmutableList.of(node)).metadata(WorkflowMetadata.builder().build()).interface_(interface_).outputs(ImmutableList.of()).build();
    client.createWorkflow(identifier, template, ImmutableMap.of());
    assertThat(stubService.createWorkflowRequest, equalTo(WorkflowOuterClass.WorkflowCreateRequest.newBuilder().setId(newIdentifier(ResourceType.WORKFLOW, WF_NAME, WF_VERSION)).setSpec(newWorkflowSpec(nodeId)).build()));
}
Also used : TypedInterface(org.flyte.api.v1.TypedInterface) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) TaskNode(org.flyte.api.v1.TaskNode) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) TaskNode(org.flyte.api.v1.TaskNode) Node(org.flyte.api.v1.Node) Test(org.junit.Test)

Example 7 with WorkflowIdentifier

use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.

the class FlyteAdminClientTest method fetchLatestWorkflowIdShouldReturnFirstWorkflowFromList.

@Test
public void fetchLatestWorkflowIdShouldReturnFirstWorkflowFromList() {
    stubService.workflowLists = Arrays.asList(WorkflowOuterClass.Workflow.newBuilder().setId(newIdentifier(ResourceType.WORKFLOW, WF_NAME, WF_VERSION)).build(), WorkflowOuterClass.Workflow.newBuilder().setId(newIdentifier(ResourceType.WORKFLOW, WF_NAME, WF_OLD_VERSION)).build());
    WorkflowIdentifier workflowId = client.fetchLatestWorkflowId(NamedEntityIdentifier.builder().project(PROJECT).domain(DOMAIN).name(WF_NAME).build());
    assertThat(workflowId, equalTo(WorkflowIdentifier.builder().project(PROJECT).domain(DOMAIN).name(WF_NAME).version(WF_VERSION).build()));
}
Also used : PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) Test(org.junit.Test)

Example 8 with WorkflowIdentifier

use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.

the class ExecuteDynamicWorkflow method execute.

private void execute() {
    Config config = Config.load();
    ExecutionConfig executionConfig = ExecutionConfig.load();
    Collection<ClassLoader> modules = ClassLoaders.forModuleDir(config.moduleDir()).values();
    Map<String, FileSystem> fileSystems = FileSystemLoader.loadFileSystems(modules);
    FileSystem outputFs = FileSystemLoader.getFileSystem(fileSystems, outputPrefix);
    ProtoWriter protoWriter = new ProtoWriter(outputPrefix, outputFs);
    try {
        FileSystem inputFs = FileSystemLoader.getFileSystem(fileSystems, inputs);
        ProtoReader protoReader = new ProtoReader(inputFs);
        TaskTemplate taskTemplate = protoReader.getTaskTemplate(taskTemplatePath);
        ClassLoader packageClassLoader = PackageLoader.load(fileSystems, taskTemplate);
        Map<String, String> env = getEnv();
        Map<WorkflowIdentifier, WorkflowTemplate> workflowTemplates = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(WorkflowTemplateRegistrar.class, env));
        Map<TaskIdentifier, RunnableTask> runnableTasks = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(RunnableTaskRegistrar.class, env));
        Map<TaskIdentifier, DynamicWorkflowTask> dynamicWorkflowTasks = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(DynamicWorkflowTaskRegistrar.class, env));
        // before we run anything, switch class loader, otherwise,
        // ServiceLoaders and other things wouldn't work, for instance,
        // FileSystemRegister in Apache Beam
        // we don't take the whole "custom" field, but only jflyte part, for that we ser-de it
        Struct custom = JFlyteCustom.deserializeFromStruct(taskTemplate.custom()).serializeToStruct();
        // all tasks already have staged jars, we can reuse 'jflyte' custom from current task to get
        // it
        Map<TaskIdentifier, TaskTemplate> taskTemplates = mapValues(ProjectClosure.createTaskTemplates(executionConfig, runnableTasks, dynamicWorkflowTasks), template -> template.toBuilder().custom(ProjectClosure.merge(template.custom(), custom)).build());
        DynamicJobSpec futures = withClassLoader(packageClassLoader, () -> {
            Map<String, Literal> input = protoReader.getInput(inputs);
            DynamicWorkflowTask task = getDynamicWorkflowTask(this.task);
            return task.run(input);
        });
        DynamicJobSpec rewrittenFutures = rewrite(executionConfig, futures, taskTemplates, workflowTemplates);
        if (rewrittenFutures.nodes().isEmpty()) {
            Map<String, Literal> outputs = getLiteralMap(rewrittenFutures.outputs());
            protoWriter.writeOutputs(outputs);
        } else {
            protoWriter.writeFutures(rewrittenFutures);
        }
    } catch (ContainerError e) {
        LOG.error("failed to run dynamic workflow", e);
        protoWriter.writeError(ProtoUtil.serializeContainerError(e));
    } catch (Throwable e) {
        LOG.error("failed to run dynamic workflow", e);
        protoWriter.writeError(ProtoUtil.serializeThrowable(e));
    }
}
Also used : DynamicWorkflowTaskRegistrar(org.flyte.api.v1.DynamicWorkflowTaskRegistrar) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) TaskIdentifier(org.flyte.api.v1.TaskIdentifier) PartialTaskIdentifier(org.flyte.api.v1.PartialTaskIdentifier) Struct(org.flyte.api.v1.Struct) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) RunnableTaskRegistrar(org.flyte.api.v1.RunnableTaskRegistrar) FileSystem(org.flyte.jflyte.api.FileSystem) Literal(org.flyte.api.v1.Literal) ClassLoaders.withClassLoader(org.flyte.jflyte.ClassLoaders.withClassLoader) TaskTemplate(org.flyte.api.v1.TaskTemplate) WorkflowTemplateRegistrar(org.flyte.api.v1.WorkflowTemplateRegistrar) DynamicWorkflowTask(org.flyte.api.v1.DynamicWorkflowTask) RunnableTask(org.flyte.api.v1.RunnableTask) DynamicJobSpec(org.flyte.api.v1.DynamicJobSpec) ContainerError(org.flyte.api.v1.ContainerError)

Example 9 with WorkflowIdentifier

use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.

the class ProtoUtilTest method shouldDeserializeWorkflowId.

@Test
void shouldDeserializeWorkflowId() {
    WorkflowIdentifier workflowId = ProtoUtil.deserializeWorkflowId(IdentifierOuterClass.Identifier.newBuilder().setResourceType(WORKFLOW).setProject(PROJECT).setDomain(DOMAIN).setName(WORKFLOW_NAME).setVersion(VERSION).build());
    assertThat(workflowId, equalTo(WorkflowIdentifier.builder().project(PROJECT).domain(DOMAIN).name(WORKFLOW_NAME).version(VERSION).build()));
}
Also used : PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 10 with WorkflowIdentifier

use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.

the class ProjectClosureTest method testCheckCycles.

@Test
public void testCheckCycles() {
    TypedInterface emptyInterface = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
    WorkflowMetadata emptyMetadata = WorkflowMetadata.builder().build();
    PartialWorkflowIdentifier rewrittenWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    PartialWorkflowIdentifier otherRewrittenWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
    WorkflowIdentifier workflowRef = WorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
    WorkflowIdentifier otherWorkflowRef = WorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
    WorkflowNode workflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenWorkflowRef)).build();
    WorkflowNode otherWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(otherRewrittenWorkflowRef)).build();
    WorkflowTemplate workflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("sub-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(otherWorkflowNode).build())).outputs(ImmutableList.of()).build();
    WorkflowTemplate otherWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("sub-2").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build())).outputs(ImmutableList.of()).build();
    Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows = ImmutableMap.of(workflowRef, workflowTemplate, otherWorkflowRef, otherWorkflowTemplate);
    // workflow -> otherWorkflow -> workflow
    IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> ProjectClosure.checkCycles(allWorkflows));
    assertEquals("Workflow [WorkflowIdentifier{name=name, domain=domain, project=project, version=version}] " + "cannot have itself as a node, directly or indirectly", exception.getMessage());
}
Also used : TypedInterface(org.flyte.api.v1.TypedInterface) WorkflowMetadata(org.flyte.api.v1.WorkflowMetadata) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowIdentifier(org.flyte.api.v1.WorkflowIdentifier) PartialWorkflowIdentifier(org.flyte.api.v1.PartialWorkflowIdentifier) WorkflowTemplate(org.flyte.api.v1.WorkflowTemplate) WorkflowNode(org.flyte.api.v1.WorkflowNode) Test(org.junit.jupiter.api.Test)

Aggregations

WorkflowIdentifier (org.flyte.api.v1.WorkflowIdentifier)11 PartialWorkflowIdentifier (org.flyte.api.v1.PartialWorkflowIdentifier)10 WorkflowTemplate (org.flyte.api.v1.WorkflowTemplate)6 Test (org.junit.jupiter.api.Test)4 Node (org.flyte.api.v1.Node)3 TypedInterface (org.flyte.api.v1.TypedInterface)3 WorkflowNode (org.flyte.api.v1.WorkflowNode)3 Test (org.junit.Test)3 WorkflowMetadata (org.flyte.api.v1.WorkflowMetadata)2 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)2 IdentifierOuterClass (flyteidl.core.IdentifierOuterClass)1 HashMap (java.util.HashMap)1 BooleanExpression (org.flyte.api.v1.BooleanExpression)1 BranchNode (org.flyte.api.v1.BranchNode)1 ContainerError (org.flyte.api.v1.ContainerError)1 DynamicJobSpec (org.flyte.api.v1.DynamicJobSpec)1 DynamicWorkflowTask (org.flyte.api.v1.DynamicWorkflowTask)1 DynamicWorkflowTaskRegistrar (org.flyte.api.v1.DynamicWorkflowTaskRegistrar)1 Literal (org.flyte.api.v1.Literal)1 Operand (org.flyte.api.v1.Operand)1