use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.
the class FlyteAdminClientTest method shouldPropagateCreateWorkflowToStub.
@Test
public void shouldPropagateCreateWorkflowToStub() {
String nodeId = "node";
WorkflowIdentifier identifier = WorkflowIdentifier.builder().domain(DOMAIN).project(PROJECT).name(WF_NAME).version(WF_VERSION).build();
TaskNode taskNode = TaskNode.builder().referenceId(PartialTaskIdentifier.builder().domain(DOMAIN).project(PROJECT).name(TASK_NAME).version(TASK_VERSION).build()).build();
Node node = Node.builder().id(nodeId).taskNode(taskNode).inputs(ImmutableList.of(Binding.builder().var_(VAR_NAME).binding(BindingData.ofScalar(Scalar.ofPrimitive(Primitive.ofStringValue(SCALAR)))).build())).upstreamNodeIds(emptyList()).build();
TypedInterface interface_ = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
WorkflowTemplate template = WorkflowTemplate.builder().nodes(ImmutableList.of(node)).metadata(WorkflowMetadata.builder().build()).interface_(interface_).outputs(ImmutableList.of()).build();
client.createWorkflow(identifier, template, ImmutableMap.of());
assertThat(stubService.createWorkflowRequest, equalTo(WorkflowOuterClass.WorkflowCreateRequest.newBuilder().setId(newIdentifier(ResourceType.WORKFLOW, WF_NAME, WF_VERSION)).setSpec(newWorkflowSpec(nodeId)).build()));
}
use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.
the class FlyteAdminClientTest method fetchLatestWorkflowIdShouldReturnFirstWorkflowFromList.
@Test
public void fetchLatestWorkflowIdShouldReturnFirstWorkflowFromList() {
stubService.workflowLists = Arrays.asList(WorkflowOuterClass.Workflow.newBuilder().setId(newIdentifier(ResourceType.WORKFLOW, WF_NAME, WF_VERSION)).build(), WorkflowOuterClass.Workflow.newBuilder().setId(newIdentifier(ResourceType.WORKFLOW, WF_NAME, WF_OLD_VERSION)).build());
WorkflowIdentifier workflowId = client.fetchLatestWorkflowId(NamedEntityIdentifier.builder().project(PROJECT).domain(DOMAIN).name(WF_NAME).build());
assertThat(workflowId, equalTo(WorkflowIdentifier.builder().project(PROJECT).domain(DOMAIN).name(WF_NAME).version(WF_VERSION).build()));
}
use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.
the class ExecuteDynamicWorkflow method execute.
private void execute() {
Config config = Config.load();
ExecutionConfig executionConfig = ExecutionConfig.load();
Collection<ClassLoader> modules = ClassLoaders.forModuleDir(config.moduleDir()).values();
Map<String, FileSystem> fileSystems = FileSystemLoader.loadFileSystems(modules);
FileSystem outputFs = FileSystemLoader.getFileSystem(fileSystems, outputPrefix);
ProtoWriter protoWriter = new ProtoWriter(outputPrefix, outputFs);
try {
FileSystem inputFs = FileSystemLoader.getFileSystem(fileSystems, inputs);
ProtoReader protoReader = new ProtoReader(inputFs);
TaskTemplate taskTemplate = protoReader.getTaskTemplate(taskTemplatePath);
ClassLoader packageClassLoader = PackageLoader.load(fileSystems, taskTemplate);
Map<String, String> env = getEnv();
Map<WorkflowIdentifier, WorkflowTemplate> workflowTemplates = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(WorkflowTemplateRegistrar.class, env));
Map<TaskIdentifier, RunnableTask> runnableTasks = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(RunnableTaskRegistrar.class, env));
Map<TaskIdentifier, DynamicWorkflowTask> dynamicWorkflowTasks = ClassLoaders.withClassLoader(packageClassLoader, () -> Registrars.loadAll(DynamicWorkflowTaskRegistrar.class, env));
// before we run anything, switch class loader, otherwise,
// ServiceLoaders and other things wouldn't work, for instance,
// FileSystemRegister in Apache Beam
// we don't take the whole "custom" field, but only jflyte part, for that we ser-de it
Struct custom = JFlyteCustom.deserializeFromStruct(taskTemplate.custom()).serializeToStruct();
// all tasks already have staged jars, we can reuse 'jflyte' custom from current task to get
// it
Map<TaskIdentifier, TaskTemplate> taskTemplates = mapValues(ProjectClosure.createTaskTemplates(executionConfig, runnableTasks, dynamicWorkflowTasks), template -> template.toBuilder().custom(ProjectClosure.merge(template.custom(), custom)).build());
DynamicJobSpec futures = withClassLoader(packageClassLoader, () -> {
Map<String, Literal> input = protoReader.getInput(inputs);
DynamicWorkflowTask task = getDynamicWorkflowTask(this.task);
return task.run(input);
});
DynamicJobSpec rewrittenFutures = rewrite(executionConfig, futures, taskTemplates, workflowTemplates);
if (rewrittenFutures.nodes().isEmpty()) {
Map<String, Literal> outputs = getLiteralMap(rewrittenFutures.outputs());
protoWriter.writeOutputs(outputs);
} else {
protoWriter.writeFutures(rewrittenFutures);
}
} catch (ContainerError e) {
LOG.error("failed to run dynamic workflow", e);
protoWriter.writeError(ProtoUtil.serializeContainerError(e));
} catch (Throwable e) {
LOG.error("failed to run dynamic workflow", e);
protoWriter.writeError(ProtoUtil.serializeThrowable(e));
}
}
use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.
the class ProtoUtilTest method shouldDeserializeWorkflowId.
@Test
void shouldDeserializeWorkflowId() {
WorkflowIdentifier workflowId = ProtoUtil.deserializeWorkflowId(IdentifierOuterClass.Identifier.newBuilder().setResourceType(WORKFLOW).setProject(PROJECT).setDomain(DOMAIN).setName(WORKFLOW_NAME).setVersion(VERSION).build());
assertThat(workflowId, equalTo(WorkflowIdentifier.builder().project(PROJECT).domain(DOMAIN).name(WORKFLOW_NAME).version(VERSION).build()));
}
use of org.flyte.api.v1.WorkflowIdentifier in project flytekit-java by flyteorg.
the class ProjectClosureTest method testCheckCycles.
@Test
public void testCheckCycles() {
TypedInterface emptyInterface = TypedInterface.builder().inputs(ImmutableMap.of()).outputs(ImmutableMap.of()).build();
WorkflowMetadata emptyMetadata = WorkflowMetadata.builder().build();
PartialWorkflowIdentifier rewrittenWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
PartialWorkflowIdentifier otherRewrittenWorkflowRef = PartialWorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
WorkflowIdentifier workflowRef = WorkflowIdentifier.builder().project("project").name("name").version("version").domain("domain").build();
WorkflowIdentifier otherWorkflowRef = WorkflowIdentifier.builder().project("project").name("other-name").version("version").domain("domain").build();
WorkflowNode workflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(rewrittenWorkflowRef)).build();
WorkflowNode otherWorkflowNode = WorkflowNode.builder().reference(WorkflowNode.Reference.ofSubWorkflowRef(otherRewrittenWorkflowRef)).build();
WorkflowTemplate workflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("sub-1").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(otherWorkflowNode).build())).outputs(ImmutableList.of()).build();
WorkflowTemplate otherWorkflowTemplate = WorkflowTemplate.builder().interface_(emptyInterface).metadata(emptyMetadata).nodes(ImmutableList.of(Node.builder().id("sub-2").inputs(ImmutableList.of()).upstreamNodeIds(ImmutableList.of()).workflowNode(workflowNode).build())).outputs(ImmutableList.of()).build();
Map<WorkflowIdentifier, WorkflowTemplate> allWorkflows = ImmutableMap.of(workflowRef, workflowTemplate, otherWorkflowRef, otherWorkflowTemplate);
// workflow -> otherWorkflow -> workflow
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> ProjectClosure.checkCycles(allWorkflows));
assertEquals("Workflow [WorkflowIdentifier{name=name, domain=domain, project=project, version=version}] " + "cannot have itself as a node, directly or indirectly", exception.getMessage());
}
Aggregations