Examples with SubWorkflow com.netflix.conductor.core.execution.tasks.SubWorkflow used on opensource projects

Example 11 with SubWorkflow

use of com.netflix.conductor.core.execution.tasks.SubWorkflow in project conductor by Netflix.

the class TestWorkflowExecutor method testUpdateParentWorkflowTask.

@Test
public void testUpdateParentWorkflowTask() {
    String parentWorkflowTaskId = "parent_workflow_task_id";
    String workflowId = "workflow_id";
    Workflow subWorkflow = new Workflow();
    subWorkflow.setWorkflowId(workflowId);
    subWorkflow.setParentWorkflowTaskId(parentWorkflowTaskId);
    subWorkflow.setStatus(WorkflowStatus.COMPLETED);
    Task subWorkflowTask = new Task();
    subWorkflowTask.setSubWorkflowId(workflowId);
    subWorkflowTask.setStatus(Status.IN_PROGRESS);
    subWorkflowTask.setExternalOutputPayloadStoragePath(null);
    when(executionDAOFacade.getTaskById(parentWorkflowTaskId)).thenReturn(subWorkflowTask);
    when(executionDAOFacade.getWorkflowById(workflowId, false)).thenReturn(subWorkflow);
    workflowExecutor.updateParentWorkflowTask(subWorkflow);
    ArgumentCaptor<Task> argumentCaptor = ArgumentCaptor.forClass(Task.class);
    verify(executionDAOFacade, times(1)).updateTask(argumentCaptor.capture());
    assertEquals(Status.COMPLETED, argumentCaptor.getAllValues().get(0).getStatus());
    assertEquals(workflowId, argumentCaptor.getAllValues().get(0).getSubWorkflowId());
}

Example 12 with SubWorkflow

use of com.netflix.conductor.core.execution.tasks.SubWorkflow in project conductor by Netflix.

the class AbstractWorkflowServiceTest method testSimpleWorkflowWithAllTaskInOneDomain.

@Test
public void testSimpleWorkflowWithAllTaskInOneDomain() throws Exception {
    clearWorkflows();
    createWorkflowWithSubWorkflow();
    WorkflowDef def = metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2_SW, 1);
    String correlationId = "unit_test_sw";
    Map<String, Object> input = new HashMap<String, Object>();
    String inputParam1 = "p1 value";
    input.put("param1", inputParam1);
    input.put("param2", "p2 value");
    Map<String, String> taskToDomain = new HashMap<String, String>();
    taskToDomain.put("*", "domain11,, domain12");
    // Poll before so that a polling for this task is "active"
    Task task = workflowExecutionService.poll("junit_task_3", "task1.junit.worker", "domain11");
    assertNull(task);
    task = workflowExecutionService.poll("junit_task_2", "task1.junit.worker", "domain12");
    assertNull(task);
    String workflowId = startOrLoadWorkflowExecution("simpleWorkflowWithTasksInOneDomain", LINEAR_WORKFLOW_T1_T2_SW, 1, correlationId, input, null, taskToDomain);
    assertNotNull(workflowId);
    Workflow workflow = workflowExecutor.getWorkflow(workflowId, false);
    assertNotNull(workflow);
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(workflow.getReasonForIncompletion(), RUNNING, workflow.getStatus());
    assertEquals(RUNNING, workflow.getStatus());
    // The very first task is the one that should be scheduled.
    assertEquals(1, workflow.getTasks().size());
    // Check Size
    Map<String, Integer> sizes = workflowExecutionService.getTaskQueueSizes(Arrays.asList("domain11:junit_task_3", "junit_task_3"));
    assertEquals(sizes.get("domain11:junit_task_3").intValue(), 1);
    assertEquals(sizes.get("junit_task_3").intValue(), 0);
    // Polling for the first task
    task = workflowExecutionService.poll("junit_task_3", "task1.junit.worker");
    assertNull(task);
    task = workflowExecutionService.poll("junit_task_3", "task1.junit.worker", "domain11");
    assertNotNull(task);
    assertEquals("junit_task_3", task.getTaskType());
    assertEquals("domain11", task.getDomain());
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    assertEquals(workflowId, task.getWorkflowInstanceId());
    List<Task> tasks = workflowExecutionService.getTasks(task.getTaskType(), null, 1);
    assertNotNull(tasks);
    assertEquals(1, tasks.size());
    task = tasks.get(0);
    String task1Op = "task1.Done";
    assertEquals(workflowId, task.getWorkflowInstanceId());
    task.getOutputData().put("op", task1Op);
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertEquals(workflowId, task.getWorkflowInstanceId());
    assertNotNull(workflow);
    assertEquals(RUNNING, workflow.getStatus());
    assertEquals(2, workflow.getTasks().size());
    // Simulating SystemTaskWorkerCoordinator to execute async system tasks
    String subWorkflowTaskId = workflow.getTaskByRefName("sw1").getTaskId();
    workflowExecutor.executeSystemTask(dummySubWorkflowSystemTask, subWorkflowTaskId, 1);
    task = workflowExecutionService.poll("junit_task_1", "task1.junit.worker");
    assertNull(task);
    task = workflowExecutionService.poll("junit_task_1", "task1.junit.worker", "domain12");
    assertNotNull(task);
    assertEquals("junit_task_1", task.getTaskType());
    workflow = workflowExecutionService.getExecutionStatus(workflowId, false);
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    assertNotNull(workflow.getTaskToDomain());
    assertEquals(workflow.getTaskToDomain().size(), 1);
    task.setStatus(COMPLETED);
    task.setReasonForIncompletion("unit test failure");
    workflowExecutionService.updateTask(task);
    task = workflowExecutionService.poll("junit_task_2", "task2.junit.worker", "domain11");
    assertNull(task);
    task = workflowExecutionService.poll("junit_task_2", "task2.junit.worker", "domain12");
    assertNotNull(task);
    assertEquals("junit_task_2", task.getTaskType());
    assertEquals("domain12", task.getDomain());
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    task.setStatus(COMPLETED);
    task.setReasonForIncompletion("unit test failure");
    workflowExecutionService.updateTask(task);
    // Execute again to re-evaluate the Subworkflow task.
    workflowExecutor.executeSystemTask(dummySubWorkflowSystemTask, subWorkflowTaskId, 1);
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(WorkflowStatus.COMPLETED, workflow.getStatus());
    tasks = workflow.getTasks();
    assertNotNull(tasks);
    assertEquals(2, tasks.size());
    assertTrue("Found " + workflow.getOutput().toString(), workflow.getOutput().containsKey("o3"));
    assertEquals("task1.Done", workflow.getOutput().get("o3"));
}

Example 13 with SubWorkflow

use of com.netflix.conductor.core.execution.tasks.SubWorkflow in project conductor by Netflix.

the class AbstractWorkflowServiceTest method runWorkflowWithSubworkflow.

private String runWorkflowWithSubworkflow() throws Exception {
    clearWorkflows();
    createWorkflowWithSubWorkflow();
    metadataService.getWorkflowDef(LINEAR_WORKFLOW_T1_T2_SW, 1);
    String correlationId = "unit_test_sw";
    Map<String, Object> input = new HashMap<>();
    String inputParam1 = "p1 value";
    input.put("param1", inputParam1);
    input.put("param2", "p2 value");
    String workflowId = startOrLoadWorkflowExecution(LINEAR_WORKFLOW_T1_T2_SW, 1, correlationId, input, null, null);
    System.out.println("testSimpleWorkflow.wfid=" + workflowId);
    assertNotNull(workflowId);
    Workflow workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(RUNNING, workflow.getStatus());
    // The very first task is the one that should be scheduled.
    assertEquals(1, workflow.getTasks().size());
    // Poll for first task and execute it
    Task task = workflowExecutionService.poll("junit_task_3", "task3.junit.worker");
    assertNotNull(task);
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    task.getOutputData().put("op", "junit_task_3.done");
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(RUNNING, workflow.getStatus());
    assertEquals(2, workflow.getTasks().size());
    // Simulating SystemTaskWorkerCoordinator to execute async system tasks
    String subWorkflowTaskId = workflow.getTaskByRefName("sw1").getTaskId();
    workflowExecutor.executeSystemTask(dummySubWorkflowSystemTask, subWorkflowTaskId, 1);
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    // Get the sub workflow id
    String subWorkflowId = null;
    for (Task t : workflow.getTasks()) {
        if (t.getTaskType().equalsIgnoreCase("SUB_WORKFLOW")) {
            subWorkflowId = t.getSubWorkflowId();
        }
    }
    assertNotNull(subWorkflowId);
    Workflow subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    assertNotNull(subWorkflow);
    assertEquals(RUNNING, subWorkflow.getStatus());
    assertEquals(1, subWorkflow.getTasks().size());
    // Now the Sub workflow is triggered
    // Poll for first task of the sub workflow and execute it
    task = workflowExecutionService.poll("junit_task_1", "task1.junit.worker");
    assertNotNull(task);
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    task.getOutputData().put("op", "junit_task_1.done");
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    assertNotNull(subWorkflow);
    assertEquals(RUNNING, subWorkflow.getStatus());
    assertEquals(2, subWorkflow.getTasks().size());
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(RUNNING, workflow.getStatus());
    assertEquals(2, workflow.getTasks().size());
    // Poll for second task of the sub workflow and execute it
    task = workflowExecutionService.poll("junit_task_2", "task2.junit.worker");
    assertNotNull(task);
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    task.getOutputData().put("op", "junit_task_2.done");
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    // Now the sub workflow and the main workflow must have finished
    subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    assertNotNull(subWorkflow);
    assertEquals(WorkflowStatus.COMPLETED, subWorkflow.getStatus());
    assertEquals(2, subWorkflow.getTasks().size());
    // Execute again to re-evaluate the Subworkflow task.
    workflowExecutor.executeSystemTask(dummySubWorkflowSystemTask, subWorkflowTaskId, 1);
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(WorkflowStatus.COMPLETED, workflow.getStatus());
    assertEquals(2, workflow.getTasks().size());
    return workflowId;
}

Example 14 with SubWorkflow

use of com.netflix.conductor.core.execution.tasks.SubWorkflow in project conductor by Netflix.

the class AbstractWorkflowServiceTest method testWorkflowWithSubWorkflowUsingExternalPayloadStorage.

@Test
public void testWorkflowWithSubWorkflowUsingExternalPayloadStorage() {
    createWorkflow_TaskSubworkflowTask();
    WorkflowDef workflowDef = metadataService.getWorkflowDef(WF_T1_SWF_T2, 1);
    assertNotNull(workflowDef);
    String workflowInputPath = INITIAL_WORKFLOW_INPUT_PATH;
    String correlationId = "subwf_external_storage";
    String workflowId = workflowExecutor.startWorkflow(WF_T1_SWF_T2, 1, correlationId, null, workflowInputPath, null, null);
    assertNotNull(workflowId);
    Workflow workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertTrue("The workflow input should not be persisted", workflow.getInput().isEmpty());
    assertEquals(workflowInputPath, workflow.getExternalInputPayloadStoragePath());
    assertEquals(workflow.getReasonForIncompletion(), WorkflowStatus.RUNNING, workflow.getStatus());
    assertEquals(1, workflow.getTasks().size());
    // Polling for the first task
    Task task = workflowExecutionService.poll("junit_task_1", "junit.worker.task_1");
    assertNotNull(task);
    assertEquals("junit_task_1", task.getTaskType());
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    assertEquals(workflowId, task.getWorkflowInstanceId());
    // update first task with COMPLETED and using external payload storage for output data
    String taskOutputPath = TASK_OUTPUT_PATH;
    task.setOutputData(null);
    task.setExternalOutputPayloadStoragePath(taskOutputPath);
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertTrue("The workflow input should not be persisted", workflow.getInput().isEmpty());
    assertEquals(workflowInputPath, workflow.getExternalInputPayloadStoragePath());
    assertEquals(workflow.getReasonForIncompletion(), WorkflowStatus.RUNNING, workflow.getStatus());
    assertEquals(2, workflow.getTasks().size());
    assertTrue("The first task output should not be persisted", workflow.getTasks().get(0).getOutputData().isEmpty());
    assertTrue("The second task input should not be persisted", workflow.getTasks().get(1).getInputData().isEmpty());
    assertEquals(taskOutputPath, workflow.getTasks().get(0).getExternalOutputPayloadStoragePath());
    assertEquals(INPUT_PAYLOAD_PATH, workflow.getTasks().get(1).getExternalInputPayloadStoragePath());
    assertEquals(SUB_WORKFLOW.name(), workflow.getTasks().get(1).getTaskType());
    // Simulating SystemTaskWorkerCoordinator to execute async system tasks
    String subWorkflowTaskId = workflow.getTaskByRefName("swt").getTaskId();
    workflowExecutor.executeSystemTask(dummySubWorkflowSystemTask, subWorkflowTaskId, 1);
    // Polling for the task within the sub_workflow
    task = workflowExecutionService.poll("junit_task_3", "task3.junit.worker");
    assertNotNull(task);
    assertEquals("junit_task_3", task.getTaskType());
    assertEquals(IN_PROGRESS, task.getStatus());
    assertEquals("TEST_SAMPLE", task.getInputData().get("p1"));
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    // Get the sub-workflow
    String subWorkflowId = task.getWorkflowInstanceId();
    Workflow subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    assertNotNull(subWorkflow);
    assertEquals(workflowId, subWorkflow.getParentWorkflowId());
    assertTrue("The sub-workflow input should not be persisted", subWorkflow.getInput().isEmpty());
    assertEquals(INPUT_PAYLOAD_PATH, subWorkflow.getExternalInputPayloadStoragePath());
    assertEquals(subWorkflow.getReasonForIncompletion(), WorkflowStatus.RUNNING, subWorkflow.getStatus());
    assertEquals(1, subWorkflow.getTasks().size());
    // update the task within sub-workflow to COMPLETED
    taskOutputPath = TASK_OUTPUT_PATH;
    task.setOutputData(null);
    task.setExternalOutputPayloadStoragePath(taskOutputPath);
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    // Execute again to re-evaluate the Subworkflow task.
    workflowExecutor.executeSystemTask(dummySubWorkflowSystemTask, subWorkflowTaskId, 1);
    // check the sub workflow
    subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    assertNotNull(subWorkflow);
    assertEquals(1, subWorkflow.getTasks().size());
    assertEquals(WorkflowStatus.COMPLETED, subWorkflow.getStatus());
    assertTrue(subWorkflow.getOutput().isEmpty());
    assertNotNull(subWorkflow.getExternalOutputPayloadStoragePath());
    assertEquals(WORKFLOW_OUTPUT_PATH, subWorkflow.getExternalOutputPayloadStoragePath());
    // check the workflow
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(WorkflowStatus.RUNNING, workflow.getStatus());
    assertEquals(3, workflow.getTasks().size());
    // Check if subworkflow task has external payload path copied from subworkflow
    Task subWorkflowTask = workflow.getTasks().stream().filter(wtask -> wtask.getTaskType().equals(SUB_WORKFLOW.name())).collect(Collectors.toList()).get(0);
    assertEquals(subWorkflowTask.getStatus(), COMPLETED);
    assertTrue(subWorkflowTask.getOutputData().isEmpty());
    assertNotNull(subWorkflowTask.getExternalOutputPayloadStoragePath());
    // Polling for the last task
    task = workflowExecutionService.poll("junit_task_2", "junit.worker.task_2");
    assertNotNull(task);
    assertEquals("junit_task_2", task.getTaskType());
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    assertEquals(workflowId, task.getWorkflowInstanceId());
    assertTrue("The task input should not be persisted", task.getInputData().isEmpty());
    assertEquals(INPUT_PAYLOAD_PATH, task.getExternalInputPayloadStoragePath());
    // update last task with COMPLETED and using external payload storage for output data
    taskOutputPath = TASK_OUTPUT_PATH;
    task.setOutputData(null);
    task.setExternalOutputPayloadStoragePath(taskOutputPath);
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    // check the parent workflow
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(WorkflowStatus.COMPLETED, workflow.getStatus());
    assertEquals(3, workflow.getTasks().size());
    assertTrue(workflow.getOutput().isEmpty());
    assertNotNull(workflow.getExternalOutputPayloadStoragePath());
    assertEquals(WORKFLOW_OUTPUT_PATH, workflow.getExternalOutputPayloadStoragePath());
}

Example 15 with SubWorkflow

use of com.netflix.conductor.core.execution.tasks.SubWorkflow in project conductor by Netflix.

the class AbstractWorkflowServiceTest method testWorkflowRerunWithSubWorkflows.

@Test
public void testWorkflowRerunWithSubWorkflows() throws Exception {
    // Execute a workflow with sub-workflow
    String workflowId = this.runWorkflowWithSubworkflow();
    // Check it completed
    Workflow workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(WorkflowStatus.COMPLETED, workflow.getStatus());
    assertEquals(2, workflow.getTasks().size());
    // Now lets pickup the first task in the sub workflow and rerun it from there
    String subWorkflowId = null;
    for (Task task : workflow.getTasks()) {
        if (task.getTaskType().equalsIgnoreCase(SubWorkflow.NAME)) {
            subWorkflowId = task.getSubWorkflowId();
        }
    }
    assertNotNull(subWorkflowId);
    Workflow subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    Task subWorkflowTask1 = null;
    for (Task task : subWorkflow.getTasks()) {
        if (task.getTaskDefName().equalsIgnoreCase("junit_task_1")) {
            subWorkflowTask1 = task;
        }
    }
    assertNotNull(subWorkflowTask1);
    RerunWorkflowRequest rerunWorkflowRequest = new RerunWorkflowRequest();
    Map<String, Object> newInput = new HashMap<>();
    newInput.put("p1", "1");
    newInput.put("p2", "2");
    rerunWorkflowRequest.setTaskInput(newInput);
    String correlationId = "unit_test_sw_new";
    Map<String, Object> input = new HashMap<>();
    input.put("param1", "New p1 value");
    input.put("param2", "New p2 value");
    rerunWorkflowRequest.setCorrelationId(correlationId);
    rerunWorkflowRequest.setWorkflowInput(input);
    rerunWorkflowRequest.setReRunFromWorkflowId(workflowId);
    rerunWorkflowRequest.setReRunFromTaskId(subWorkflowTask1.getTaskId());
    // Rerun
    workflowExecutor.rerun(rerunWorkflowRequest);
    // The main WF and the sub WF should be in RUNNING state
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(RUNNING, workflow.getStatus());
    assertEquals(2, workflow.getTasks().size());
    assertEquals(correlationId, workflow.getCorrelationId());
    assertEquals("New p1 value", workflow.getInput().get("param1"));
    assertEquals("New p2 value", workflow.getInput().get("param2"));
    subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    assertNotNull(subWorkflow);
    assertEquals(RUNNING, subWorkflow.getStatus());
    // Since we are re running from the sub workflow task, there
    // should be only 1 task that is SCHEDULED
    assertEquals(1, subWorkflow.getTasks().size());
    assertEquals(SCHEDULED, subWorkflow.getTasks().get(0).getStatus());
    // Now execute the task
    Task task = workflowExecutionService.poll("junit_task_1", "task1.junit.worker");
    assertNotNull(task);
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    assertEquals(task.getInputData().get("p1").toString(), "1");
    assertEquals(task.getInputData().get("p2").toString(), "2");
    task.getOutputData().put("op", "junit_task_1.done");
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    assertNotNull(subWorkflow);
    assertEquals(RUNNING, subWorkflow.getStatus());
    assertEquals(2, subWorkflow.getTasks().size());
    // Poll for second task of the sub workflow and execute it
    task = workflowExecutionService.poll("junit_task_2", "task2.junit.worker");
    assertNotNull(task);
    assertTrue(workflowExecutionService.ackTaskReceived(task.getTaskId()));
    task.getOutputData().put("op", "junit_task_2.done");
    task.setStatus(COMPLETED);
    workflowExecutionService.updateTask(task);
    // Now the sub workflow and the main workflow must have finished
    subWorkflow = workflowExecutionService.getExecutionStatus(subWorkflowId, true);
    assertNotNull(subWorkflow);
    assertEquals(WorkflowStatus.COMPLETED, subWorkflow.getStatus());
    assertEquals(2, subWorkflow.getTasks().size());
    // Execute again to re-evaluate the Subworkflow task.
    workflowExecutor.executeSystemTask(dummySubWorkflowSystemTask, subWorkflow.getParentWorkflowTaskId(), 1);
    workflow = workflowExecutionService.getExecutionStatus(workflowId, true);
    assertNotNull(workflow);
    assertEquals(WorkflowStatus.COMPLETED, workflow.getStatus());
    assertEquals(2, workflow.getTasks().size());
}