Search in sources :

Example 16 with TaskDef

use of com.netflix.conductor.common.metadata.tasks.TaskDef in project conductor by Netflix.

the class CassandraMetadataDAO method getTaskDefFromDB.

private TaskDef getTaskDefFromDB(String name) {
    try {
        ResultSet resultSet = session.execute(selectTaskDefStatement.bind(name));
        recordCassandraDaoRequests("getTaskDef");
        return Optional.ofNullable(resultSet.one()).map(row -> readValue(row.getString(TASK_DEFINITION_KEY), TaskDef.class)).orElse(null);
    } catch (Exception e) {
        Monitors.error(CLASS_NAME, "getTaskDef");
        String errorMsg = String.format("Failed to get task def: %s", name);
        LOGGER.error(errorMsg, e);
        throw new ApplicationException(Code.BACKEND_ERROR, errorMsg, e);
    }
}
Also used : Row(com.datastax.driver.core.Row) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) MetadataDAO(com.netflix.conductor.dao.MetadataDAO) Singleton(javax.inject.Singleton) ArrayList(java.util.ArrayList) PreparedStatement(com.datastax.driver.core.PreparedStatement) Inject(javax.inject.Inject) CassandraConfiguration(com.netflix.conductor.cassandra.CassandraConfiguration) ResultSet(com.datastax.driver.core.ResultSet) Session(com.datastax.driver.core.Session) Map(java.util.Map) Code(com.netflix.conductor.core.execution.ApplicationException.Code) TASK_DEFINITION_KEY(com.netflix.conductor.util.Constants.TASK_DEFINITION_KEY) TaskDef(com.netflix.conductor.common.metadata.tasks.TaskDef) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) Logger(org.slf4j.Logger) Trace(com.netflix.conductor.annotations.Trace) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) WorkflowDef(com.netflix.conductor.common.metadata.workflow.WorkflowDef) TASK_DEFS_KEY(com.netflix.conductor.util.Constants.TASK_DEFS_KEY) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Monitors(com.netflix.conductor.metrics.Monitors) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) WORKFLOW_DEF_NAME_VERSION_KEY(com.netflix.conductor.util.Constants.WORKFLOW_DEF_NAME_VERSION_KEY) Statements(com.netflix.conductor.util.Statements) WORKFLOW_DEFINITION_KEY(com.netflix.conductor.util.Constants.WORKFLOW_DEFINITION_KEY) Optional(java.util.Optional) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) WORKFLOW_DEF_INDEX_KEY(com.netflix.conductor.util.Constants.WORKFLOW_DEF_INDEX_KEY) Collections(java.util.Collections) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) ResultSet(com.datastax.driver.core.ResultSet) ApplicationException(com.netflix.conductor.core.execution.ApplicationException)

Example 17 with TaskDef

use of com.netflix.conductor.common.metadata.tasks.TaskDef in project conductor by Netflix.

the class TestDeciderOutcomes method testOptional.

@Test
public void testOptional() {
    WorkflowDef def = new WorkflowDef();
    def.setName("test");
    WorkflowTask task1 = new WorkflowTask();
    task1.setName("task0");
    task1.setType("SIMPLE");
    task1.setTaskReferenceName("t0");
    task1.getInputParameters().put("taskId", "${CPEWF_TASK_ID}");
    task1.setOptional(true);
    task1.setTaskDefinition(new TaskDef("task0"));
    WorkflowTask task2 = new WorkflowTask();
    task2.setName("task1");
    task2.setType("SIMPLE");
    task2.setTaskReferenceName("t1");
    task2.setTaskDefinition(new TaskDef("task1"));
    def.getTasks().add(task1);
    def.getTasks().add(task2);
    def.setSchemaVersion(2);
    Workflow workflow = new Workflow();
    workflow.setWorkflowDefinition(def);
    workflow.setStartTime(System.currentTimeMillis());
    DeciderOutcome outcome = deciderService.decide(workflow);
    assertNotNull(outcome);
    System.out.println("Schedule after starting: " + outcome.tasksToBeScheduled);
    assertEquals(1, outcome.tasksToBeScheduled.size());
    assertEquals(task1.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName());
    System.out.println("TaskId of the scheduled task in input: " + outcome.tasksToBeScheduled.get(0).getInputData());
    for (int i = 0; i < 3; i++) {
        String task1Id = outcome.tasksToBeScheduled.get(0).getTaskId();
        assertEquals(task1Id, outcome.tasksToBeScheduled.get(0).getInputData().get("taskId"));
        workflow.getTasks().clear();
        workflow.getTasks().addAll(outcome.tasksToBeScheduled);
        workflow.getTasks().get(0).setStatus(Status.FAILED);
        outcome = deciderService.decide(workflow);
        assertNotNull(outcome);
        System.out.println("Schedule: " + outcome.tasksToBeScheduled);
        System.out.println("Update: " + outcome.tasksToBeUpdated);
        assertEquals(1, outcome.tasksToBeUpdated.size());
        assertEquals(1, outcome.tasksToBeScheduled.size());
        assertEquals(Task.Status.FAILED, workflow.getTasks().get(0).getStatus());
        assertEquals(task1Id, outcome.tasksToBeUpdated.get(0).getTaskId());
        assertEquals(task1.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName());
        assertEquals(i + 1, outcome.tasksToBeScheduled.get(0).getRetryCount());
    }
    String task1Id = outcome.tasksToBeScheduled.get(0).getTaskId();
    workflow.getTasks().clear();
    workflow.getTasks().addAll(outcome.tasksToBeScheduled);
    workflow.getTasks().get(0).setStatus(Status.FAILED);
    outcome = deciderService.decide(workflow);
    assertNotNull(outcome);
    System.out.println("Schedule: " + outcome.tasksToBeScheduled);
    System.out.println("Update: " + outcome.tasksToBeUpdated);
    assertEquals(1, outcome.tasksToBeUpdated.size());
    assertEquals(1, outcome.tasksToBeScheduled.size());
    assertEquals(Task.Status.COMPLETED_WITH_ERRORS, workflow.getTasks().get(0).getStatus());
    assertEquals(task1Id, outcome.tasksToBeUpdated.get(0).getTaskId());
    assertEquals(task2.getTaskReferenceName(), outcome.tasksToBeScheduled.get(0).getReferenceTaskName());
}
Also used : WorkflowDef(com.netflix.conductor.common.metadata.workflow.WorkflowDef) TaskDef(com.netflix.conductor.common.metadata.tasks.TaskDef) Workflow(com.netflix.conductor.common.run.Workflow) WorkflowTask(com.netflix.conductor.common.metadata.workflow.WorkflowTask) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) DeciderOutcome(com.netflix.conductor.core.execution.DeciderService.DeciderOutcome) Test(org.junit.Test)

Example 18 with TaskDef

use of com.netflix.conductor.common.metadata.tasks.TaskDef in project conductor by Netflix.

the class TestDeciderOutcomes method testOptionalWithDynamicFork.

@Test
public void testOptionalWithDynamicFork() {
    WorkflowDef def = new WorkflowDef();
    def.setName("test");
    WorkflowTask task1 = new WorkflowTask();
    task1.setName("fork0");
    task1.setWorkflowTaskType(TaskType.FORK_JOIN_DYNAMIC);
    task1.setTaskReferenceName("fork0");
    task1.setDynamicForkTasksInputParamName("forkedInputs");
    task1.setDynamicForkTasksParam("forks");
    task1.getInputParameters().put("forks", "${workflow.input.forks}");
    task1.getInputParameters().put("forkedInputs", "${workflow.input.forkedInputs}");
    WorkflowTask task2 = new WorkflowTask();
    task2.setName("join0");
    task2.setType("JOIN");
    task2.setTaskReferenceName("join0");
    def.getTasks().add(task1);
    def.getTasks().add(task2);
    def.setSchemaVersion(2);
    Workflow workflow = new Workflow();
    workflow.setWorkflowDefinition(def);
    List<WorkflowTask> forks = new LinkedList<>();
    Map<String, Map<String, Object>> forkedInputs = new HashMap<>();
    for (int i = 0; i < 3; i++) {
        WorkflowTask workflowTask = new WorkflowTask();
        workflowTask.setName("f" + i);
        workflowTask.setTaskReferenceName("f" + i);
        workflowTask.setWorkflowTaskType(TaskType.SIMPLE);
        workflowTask.setOptional(true);
        workflowTask.setTaskDefinition(new TaskDef("f" + i));
        forks.add(workflowTask);
        forkedInputs.put(workflowTask.getTaskReferenceName(), new HashMap<>());
    }
    workflow.getInput().put("forks", forks);
    workflow.getInput().put("forkedInputs", forkedInputs);
    workflow.setStartTime(System.currentTimeMillis());
    DeciderOutcome outcome = deciderService.decide(workflow);
    assertNotNull(outcome);
    assertEquals(5, outcome.tasksToBeScheduled.size());
    assertEquals(0, outcome.tasksToBeUpdated.size());
    assertEquals(SystemTaskType.FORK.name(), outcome.tasksToBeScheduled.get(0).getTaskType());
    assertEquals(Task.Status.COMPLETED, outcome.tasksToBeScheduled.get(0).getStatus());
    for (int retryCount = 0; retryCount < 4; retryCount++) {
        for (Task taskToBeScheduled : outcome.tasksToBeScheduled) {
            if (taskToBeScheduled.getTaskDefName().equals("join0")) {
                assertEquals(Task.Status.IN_PROGRESS, taskToBeScheduled.getStatus());
            } else if (taskToBeScheduled.getTaskType().matches("(f0|f1|f2)")) {
                assertEquals(Task.Status.SCHEDULED, taskToBeScheduled.getStatus());
                taskToBeScheduled.setStatus(Status.FAILED);
            }
            taskToBeScheduled.setUpdateTime(System.currentTimeMillis());
        }
        workflow.getTasks().addAll(outcome.tasksToBeScheduled);
        outcome = deciderService.decide(workflow);
        assertNotNull(outcome);
    }
    assertEquals(SystemTaskType.JOIN.name(), outcome.tasksToBeScheduled.get(0).getTaskType());
    for (int i = 0; i < 3; i++) {
        assertEquals(Task.Status.COMPLETED_WITH_ERRORS, outcome.tasksToBeUpdated.get(i).getStatus());
        assertEquals("f" + (i), outcome.tasksToBeUpdated.get(i).getTaskDefName());
    }
    assertEquals(Task.Status.IN_PROGRESS, outcome.tasksToBeScheduled.get(0).getStatus());
    new Join().execute(workflow, outcome.tasksToBeScheduled.get(0), null);
    assertEquals(Task.Status.COMPLETED, outcome.tasksToBeScheduled.get(0).getStatus());
    outcome.tasksToBeScheduled.stream().map(task -> task.getStatus() + ":" + task.getTaskType() + ":").forEach(System.out::println);
    outcome.tasksToBeUpdated.stream().map(task -> task.getStatus() + ":" + task.getTaskType() + ":").forEach(System.out::println);
}
Also used : Arrays(java.util.Arrays) TaskMapper(com.netflix.conductor.core.execution.mapper.TaskMapper) Join(com.netflix.conductor.core.execution.tasks.Join) ForkJoinDynamicTaskMapper(com.netflix.conductor.core.execution.mapper.ForkJoinDynamicTaskMapper) Assert.assertNotSame(org.junit.Assert.assertNotSame) Status(com.netflix.conductor.common.metadata.tasks.Task.Status) HashMap(java.util.HashMap) MetadataDAO(com.netflix.conductor.dao.MetadataDAO) Task(com.netflix.conductor.common.metadata.tasks.Task) SimpleTaskMapper(com.netflix.conductor.core.execution.mapper.SimpleTaskMapper) DeserializationFeature(com.fasterxml.jackson.databind.DeserializationFeature) ExternalPayloadStorageUtils(com.netflix.conductor.core.utils.ExternalPayloadStorageUtils) UserDefinedTaskMapper(com.netflix.conductor.core.execution.mapper.UserDefinedTaskMapper) WaitTaskMapper(com.netflix.conductor.core.execution.mapper.WaitTaskMapper) Workflow(com.netflix.conductor.common.run.Workflow) Map(java.util.Map) JoinTaskMapper(com.netflix.conductor.core.execution.mapper.JoinTaskMapper) LinkedList(java.util.LinkedList) Before(org.junit.Before) ForkJoinTaskMapper(com.netflix.conductor.core.execution.mapper.ForkJoinTaskMapper) TaskDef(com.netflix.conductor.common.metadata.tasks.TaskDef) SubWorkflowTaskMapper(com.netflix.conductor.core.execution.mapper.SubWorkflowTaskMapper) HTTPTaskMapper(com.netflix.conductor.core.execution.mapper.HTTPTaskMapper) DecisionTaskMapper(com.netflix.conductor.core.execution.mapper.DecisionTaskMapper) DynamicTaskMapper(com.netflix.conductor.core.execution.mapper.DynamicTaskMapper) Assert.assertNotNull(org.junit.Assert.assertNotNull) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Assert.assertTrue(org.junit.Assert.assertTrue) WorkflowDef(com.netflix.conductor.common.metadata.workflow.WorkflowDef) Include(com.fasterxml.jackson.annotation.JsonInclude.Include) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) WorkflowTask(com.netflix.conductor.common.metadata.workflow.WorkflowTask) DeciderOutcome(com.netflix.conductor.core.execution.DeciderService.DeciderOutcome) List(java.util.List) Assert.assertFalse(org.junit.Assert.assertFalse) TaskType(com.netflix.conductor.common.metadata.workflow.TaskType) EventTaskMapper(com.netflix.conductor.core.execution.mapper.EventTaskMapper) Configuration(com.netflix.conductor.core.config.Configuration) JsonMapperProvider(com.netflix.conductor.common.utils.JsonMapperProvider) Assert.assertEquals(org.junit.Assert.assertEquals) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Mockito.mock(org.mockito.Mockito.mock) InputStream(java.io.InputStream) Task(com.netflix.conductor.common.metadata.tasks.Task) WorkflowTask(com.netflix.conductor.common.metadata.workflow.WorkflowTask) WorkflowDef(com.netflix.conductor.common.metadata.workflow.WorkflowDef) HashMap(java.util.HashMap) Workflow(com.netflix.conductor.common.run.Workflow) Join(com.netflix.conductor.core.execution.tasks.Join) WorkflowTask(com.netflix.conductor.common.metadata.workflow.WorkflowTask) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) LinkedList(java.util.LinkedList) TaskDef(com.netflix.conductor.common.metadata.tasks.TaskDef) HashMap(java.util.HashMap) Map(java.util.Map) DeciderOutcome(com.netflix.conductor.core.execution.DeciderService.DeciderOutcome) Test(org.junit.Test)

Example 19 with TaskDef

use of com.netflix.conductor.common.metadata.tasks.TaskDef in project conductor by Netflix.

the class TestDeciderService method testCustomRetryWithWorkflowTask.

@Test
public void testCustomRetryWithWorkflowTask() {
    Workflow workflow = createDefaultWorkflow();
    Task task = new Task();
    task.setStatus(Status.FAILED);
    task.setTaskId("t1");
    task.setStartDelayInSeconds(30);
    TaskDef taskDef = new TaskDef();
    taskDef.setRetryDelaySeconds(60);
    taskDef.setRetryLogic(TaskDef.RetryLogic.CUSTOM);
    WorkflowTask workflowTask = new WorkflowTask();
    workflowTask.setRetryLogic(RetryLogic.FIXED);
    workflowTask.setStartDelay(80);
    // Retry delay from the task as tasDef retry policy is CUSTOM,
    // but workflow task would be preferred which is FIXED (retryDelay will come from workflowTask)
    Optional<Task> task2 = deciderService.retry(taskDef, workflowTask, task, workflow);
    assertEquals(80, task2.get().getStartDelayInSeconds());
    // Retry delay from the task as tasDef retry policy is CUSTOM,
    // but workflow task would be preferred which is CUSTOM (retryDelay will be 0 since task.retryDelay = -1)
    task2.get().setStartDelayInSeconds(-1);
    workflowTask.setRetryLogic(RetryLogic.CUSTOM);
    workflowTask.setStartDelay(90);
    // Custom retry policy from Workflow,
    Optional<Task> task3 = deciderService.retry(taskDef, workflowTask, task2.get(), workflow);
    assertEquals(0, task3.get().getStartDelayInSeconds());
    // Retry delay from the task as tasDef retry policy is CUSTOM,
    // workflow task would be preferred which is CUSTOM (retryDelay will come from the workflowTask,
    // since task.retryDelay = 0)
    task3.get().setStartDelayInSeconds(0);
    Optional<Task> task4 = deciderService.retry(taskDef, workflowTask, task3.get(), workflow);
    assertEquals(90, task4.get().getCallbackAfterSeconds());
}
Also used : Task(com.netflix.conductor.common.metadata.tasks.Task) WorkflowTask(com.netflix.conductor.common.metadata.workflow.WorkflowTask) TaskDef(com.netflix.conductor.common.metadata.tasks.TaskDef) Workflow(com.netflix.conductor.common.run.Workflow) WorkflowTask(com.netflix.conductor.common.metadata.workflow.WorkflowTask) Test(org.junit.Test)

Example 20 with TaskDef

use of com.netflix.conductor.common.metadata.tasks.TaskDef in project conductor by Netflix.

the class TestDeciderService method testGetTaskInputV2Partial.

@Test
public void testGetTaskInputV2Partial() {
    Workflow workflow = createDefaultWorkflow();
    System.setProperty("EC2_INSTANCE", "i-123abcdef990");
    Map<String, Object> wfi = new HashMap<>();
    Map<String, Object> wfmap = new HashMap<>();
    wfmap.put("input", workflow.getInput());
    wfmap.put("output", workflow.getOutput());
    wfi.put("workflow", wfmap);
    workflow.getTasks().stream().map(Task::getReferenceTaskName).forEach(ref -> {
        Map<String, Object> taskInput = workflow.getTaskByRefName(ref).getInputData();
        Map<String, Object> taskOutput = workflow.getTaskByRefName(ref).getOutputData();
        Map<String, Object> io = new HashMap<>();
        io.put("input", taskInput);
        io.put("output", taskOutput);
        wfi.put(ref, io);
    });
    workflow.getWorkflowDefinition().setSchemaVersion(2);
    Map<String, Object> ip = new HashMap<>();
    ip.put("workflowInputParam", "${workflow.input.requestId}");
    ip.put("workfowOutputParam", "${workflow.output.name}");
    ip.put("taskOutputParam", "${task2.output.location}");
    ip.put("taskOutputParam2", "${task2.output.locationBad}");
    ip.put("taskOutputParam3", "${task3.output.location}");
    ip.put("constParam", "Some String value   &");
    ip.put("partial", "${task2.output.location}/something?host=${EC2_INSTANCE}");
    ip.put("jsonPathExtracted", "${workflow.output.names[*].year}");
    ip.put("secondName", "${workflow.output.names[1].name}");
    ip.put("concatenatedName", "The Band is: ${workflow.output.names[1].name}-\t${EC2_INSTANCE}");
    TaskDef taskDef = new TaskDef();
    taskDef.getInputTemplate().put("opname", "${workflow.output.name}");
    List<Object> listParams = new LinkedList<>();
    List<Object> listParams2 = new LinkedList<>();
    listParams2.add("${workflow.input.requestId}-10-${EC2_INSTANCE}");
    listParams.add(listParams2);
    Map<String, Object> map = new HashMap<>();
    map.put("name", "${workflow.output.names[0].name}");
    map.put("hasAwards", "${workflow.input.hasAwards}");
    listParams.add(map);
    taskDef.getInputTemplate().put("listValues", listParams);
    Map<String, Object> taskInput = parametersUtils.getTaskInput(ip, workflow, taskDef, null);
    assertNotNull(taskInput);
    assertTrue(taskInput.containsKey("workflowInputParam"));
    assertTrue(taskInput.containsKey("taskOutputParam"));
    assertTrue(taskInput.containsKey("taskOutputParam2"));
    assertTrue(taskInput.containsKey("taskOutputParam3"));
    assertNull(taskInput.get("taskOutputParam2"));
    assertNotNull(taskInput.get("jsonPathExtracted"));
    assertTrue(taskInput.get("jsonPathExtracted") instanceof List);
    assertNotNull(taskInput.get("secondName"));
    assertTrue(taskInput.get("secondName") instanceof String);
    assertEquals("The Doors", taskInput.get("secondName"));
    assertEquals("The Band is: The Doors-\ti-123abcdef990", taskInput.get("concatenatedName"));
    assertEquals("request id 001", taskInput.get("workflowInputParam"));
    assertEquals("http://location", taskInput.get("taskOutputParam"));
    assertNull(taskInput.get("taskOutputParam3"));
    assertNotNull(taskInput.get("partial"));
    assertEquals("http://location/something?host=i-123abcdef990", taskInput.get("partial"));
}
Also used : HashMap(java.util.HashMap) TaskDef(com.netflix.conductor.common.metadata.tasks.TaskDef) Workflow(com.netflix.conductor.common.run.Workflow) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) LinkedList(java.util.LinkedList) Test(org.junit.Test)

Aggregations

TaskDef (com.netflix.conductor.common.metadata.tasks.TaskDef)172 Test (org.junit.Test)128 WorkflowTask (com.netflix.conductor.common.metadata.workflow.WorkflowTask)121 Task (com.netflix.conductor.common.metadata.tasks.Task)77 Workflow (com.netflix.conductor.common.run.Workflow)76 WorkflowDef (com.netflix.conductor.common.metadata.workflow.WorkflowDef)73 HashMap (java.util.HashMap)56 ArrayList (java.util.ArrayList)32 ConstraintViolation (javax.validation.ConstraintViolation)31 SubWorkflow (com.netflix.conductor.core.execution.tasks.SubWorkflow)30 UserTask (com.netflix.conductor.tests.utils.UserTask)28 LinkedList (java.util.LinkedList)28 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)27 List (java.util.List)22 Map (java.util.Map)19 ApplicationException (com.netflix.conductor.core.execution.ApplicationException)18 Before (org.junit.Before)14 ExpectedException (org.junit.rules.ExpectedException)13 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)12 Collectors (java.util.stream.Collectors)11