Examples with Task com.netflix.conductor.common.metadata.tasks.Task used on opensource projects

Example 91 with Task

use of com.netflix.conductor.common.metadata.tasks.Task in project conductor by Netflix.

the class ForkJoinDynamicTaskMapper method getMappedTasks.

/**
 * This method gets the list of tasks that need to scheduled when the task to scheduled is of type {@link TaskType#FORK_JOIN_DYNAMIC}.
 * Creates a Fork Task, followed by the Dynamic tasks and a final JOIN task.
 * <p>The definitions of the dynamic forks that need to be scheduled are available in the {@link WorkflowTask#getInputParameters()}
 * which are accessed using the {@link TaskMapperContext#getTaskToSchedule()}. The dynamic fork task definitions are referred by a key value either by
 * {@link WorkflowTask#getDynamicForkTasksParam()} or by {@link WorkflowTask#getDynamicForkJoinTasksParam()}
 * </p>
 * When creating the list of tasks to be scheduled a set of preconditions are validated:
 * <ul>
 * <li>If the input parameter representing the Dynamic fork tasks is available as part of {@link WorkflowTask#getDynamicForkTasksParam()} then
 * the input for the dynamic task is validated to be a map by using {@link WorkflowTask#getDynamicForkTasksInputParamName()}</li>
 * <li>If the input parameter representing the Dynamic fork tasks is available as part of {@link WorkflowTask#getDynamicForkJoinTasksParam()} then
 * the input for the dynamic tasks is available in the payload of the tasks definition.
 * </li>
 * <li>A check is performed that the next following task in the {@link WorkflowDef} is a {@link TaskType#JOIN}</li>
 * </ul>
 *
 * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link WorkflowDef}, {@link Workflow} and a string representation of the TaskId
 * @throws TerminateWorkflowException In case of:
 *                                    <ul>
 *                                    <li>
 *                                    When the task after {@link TaskType#FORK_JOIN_DYNAMIC} is not a {@link TaskType#JOIN}
 *                                    </li>
 *                                    <li>
 *                                    When the input parameters for the dynamic tasks are not of type {@link Map}
 *                                    </li>
 *                                    </ul>
 * @return: List of tasks in the following order:
 * <ul>
 * <li>
 * {@link SystemTaskType#FORK} with {@link Task.Status#COMPLETED}
 * </li>
 * <li>
 * Might be any kind of task, but this is most cases is a UserDefinedTask with {@link Task.Status#SCHEDULED}
 * </li>
 * <li>
 * {@link SystemTaskType#JOIN} with {@link Task.Status#IN_PROGRESS}
 * </li>
 * </ul>
 */
@Override
public List<Task> getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException {
    logger.debug("TaskMapperContext {} in ForkJoinDynamicTaskMapper", taskMapperContext);
    WorkflowTask taskToSchedule = taskMapperContext.getTaskToSchedule();
    Workflow workflowInstance = taskMapperContext.getWorkflowInstance();
    String taskId = taskMapperContext.getTaskId();
    int retryCount = taskMapperContext.getRetryCount();
    List<Task> mappedTasks = new LinkedList<>();
    // Get the list of dynamic tasks and the input for the tasks
    Pair<List<WorkflowTask>, Map<String, Map<String, Object>>> workflowTasksAndInputPair = Optional.ofNullable(taskToSchedule.getDynamicForkTasksParam()).map(dynamicForkTaskParam -> getDynamicForkTasksAndInput(taskToSchedule, workflowInstance, dynamicForkTaskParam)).orElseGet(() -> getDynamicForkJoinTasksAndInput(taskToSchedule, workflowInstance));
    List<WorkflowTask> dynForkTasks = workflowTasksAndInputPair.getLeft();
    Map<String, Map<String, Object>> tasksInput = workflowTasksAndInputPair.getRight();
    // Create Fork Task which needs to be followed by the dynamic tasks
    Task forkDynamicTask = createDynamicForkTask(taskToSchedule, workflowInstance, taskId, dynForkTasks);
    mappedTasks.add(forkDynamicTask);
    List<String> joinOnTaskRefs = new LinkedList<>();
    // which indicates that the following task after that needs to be a join task
    for (WorkflowTask dynForkTask : dynForkTasks) {
        // TODO this is a cyclic dependency, break it out using function composition
        List<Task> forkedTasks = taskMapperContext.getDeciderService().getTasksToBeScheduled(workflowInstance, dynForkTask, retryCount);
        // same reference name has already been created in the Workflow.
        if (forkedTasks == null || forkedTasks.isEmpty()) {
            Optional<String> existingTaskRefName = workflowInstance.getTasks().stream().filter(runningTask -> runningTask.getStatus().equals(Task.Status.IN_PROGRESS) || runningTask.getStatus().isTerminal()).map(Task::getReferenceTaskName).filter(refTaskName -> refTaskName.equals(dynForkTask.getTaskReferenceName())).findAny();
            // Construct an informative error message
            String terminateMessage = "No dynamic tasks could be created for the Workflow: " + workflowInstance.toShortString() + ", Dynamic Fork Task: " + dynForkTask;
            if (existingTaskRefName.isPresent()) {
                terminateMessage += "Attempted to create a duplicate task reference name: " + existingTaskRefName.get();
            }
            throw new TerminateWorkflowException(terminateMessage);
        }
        for (Task forkedTask : forkedTasks) {
            Map<String, Object> forkedTaskInput = tasksInput.get(forkedTask.getReferenceTaskName());
            forkedTask.getInputData().putAll(forkedTaskInput);
        }
        mappedTasks.addAll(forkedTasks);
        // Get the last of the dynamic tasks so that the join can be performed once this task is done
        Task last = forkedTasks.get(forkedTasks.size() - 1);
        joinOnTaskRefs.add(last.getReferenceTaskName());
    }
    // From the workflow definition get the next task and make sure that it is a JOIN task.
    // The dynamic fork tasks need to be followed by a join task
    WorkflowTask joinWorkflowTask = workflowInstance.getWorkflowDefinition().getNextTask(taskToSchedule.getTaskReferenceName());
    if (joinWorkflowTask == null || !joinWorkflowTask.getType().equals(TaskType.JOIN.name())) {
        throw new TerminateWorkflowException("Dynamic join definition is not followed by a join task.  Check the blueprint");
    }
    // Create Join task
    HashMap<String, Object> joinInput = new HashMap<>();
    joinInput.put("joinOn", joinOnTaskRefs);
    Task joinTask = createJoinTask(workflowInstance, joinWorkflowTask, joinInput);
    mappedTasks.add(joinTask);
    return mappedTasks;
}

Example 92 with Task

use of com.netflix.conductor.common.metadata.tasks.Task in project conductor by Netflix.

the class ForkJoinDynamicTaskMapper method createDynamicForkTask.

/**
 * This method creates a FORK task and adds the list of dynamic fork tasks keyed by "forkedTaskDefs" and
 * their names keyed by "forkedTasks" into {@link Task#getInputData()}
 *
 * @param taskToSchedule    A {@link WorkflowTask} representing {@link TaskType#FORK_JOIN_DYNAMIC}
 * @param workflowInstance: A instance of the {@link Workflow} which represents the workflow being executed.
 * @param taskId:           The string representation of {@link java.util.UUID} which will be set as the taskId.
 * @param dynForkTasks:     The list of dynamic forked tasks, the reference names of these tasks will be added to the forkDynamicTask
 * @return A new instance of {@link Task} representing a {@link SystemTaskType#FORK}
 */
@VisibleForTesting
Task createDynamicForkTask(WorkflowTask taskToSchedule, Workflow workflowInstance, String taskId, List<WorkflowTask> dynForkTasks) {
    Task forkDynamicTask = new Task();
    forkDynamicTask.setTaskType(SystemTaskType.FORK.name());
    forkDynamicTask.setTaskDefName(SystemTaskType.FORK.name());
    forkDynamicTask.setReferenceTaskName(taskToSchedule.getTaskReferenceName());
    forkDynamicTask.setWorkflowInstanceId(workflowInstance.getWorkflowId());
    forkDynamicTask.setCorrelationId(workflowInstance.getCorrelationId());
    forkDynamicTask.setScheduledTime(System.currentTimeMillis());
    forkDynamicTask.setEndTime(System.currentTimeMillis());
    List<String> forkedTaskNames = dynForkTasks.stream().map(WorkflowTask::getTaskReferenceName).collect(Collectors.toList());
    forkDynamicTask.getInputData().put("forkedTasks", forkedTaskNames);
    // TODO: Remove this parameter in the later releases
    forkDynamicTask.getInputData().put("forkedTaskDefs", dynForkTasks);
    forkDynamicTask.setTaskId(taskId);
    forkDynamicTask.setStatus(Task.Status.COMPLETED);
    forkDynamicTask.setWorkflowTask(taskToSchedule);
    forkDynamicTask.setWorkflowPriority(workflowInstance.getPriority());
    return forkDynamicTask;
}

Example 93 with Task

use of com.netflix.conductor.common.metadata.tasks.Task in project conductor by Netflix.

the class DoWhile method getEvaluatedCondition.

@VisibleForTesting
boolean getEvaluatedCondition(Workflow workflow, Task task, WorkflowExecutor workflowExecutor) throws ScriptException {
    TaskDef taskDefinition = null;
    try {
        taskDefinition = workflowExecutor.getTaskDefinition(task);
    } catch (TerminateWorkflowException e) {
    // It is ok to not have a task definition for a DO_WHILE task
    }
    Map<String, Object> taskInput = parametersUtils.getTaskInputV2(task.getWorkflowTask().getInputParameters(), workflow, task.getTaskId(), taskDefinition);
    taskInput.put(task.getReferenceTaskName(), task.getOutputData());
    List<Task> loopOver = workflow.getTasks().stream().filter(t -> (task.getWorkflowTask().has(TaskUtils.removeIterationFromTaskRefName(t.getReferenceTaskName())) && !task.getReferenceTaskName().equals(t.getReferenceTaskName()))).collect(Collectors.toList());
    for (Task loopOverTask : loopOver) {
        taskInput.put(TaskUtils.removeIterationFromTaskRefName(loopOverTask.getReferenceTaskName()), loopOverTask.getOutputData());
    }
    String condition = task.getWorkflowTask().getLoopCondition();
    boolean shouldContinue = false;
    if (condition != null) {
        logger.debug("Condition: {} is being evaluated", condition);
        // Evaluate the expression by using the Nashhorn based script evaluator
        shouldContinue = ScriptEvaluator.evalBool(condition, taskInput);
    }
    return shouldContinue;
}

Example 94 with Task

use of com.netflix.conductor.common.metadata.tasks.Task in project conductor by Netflix.

the class Join method execute.

@Override
@SuppressWarnings("unchecked")
public boolean execute(Workflow workflow, Task task, WorkflowExecutor provider) {
    boolean allDone = true;
    boolean hasFailures = false;
    StringBuilder failureReason = new StringBuilder();
    List<String> joinOn = (List<String>) task.getInputData().get("joinOn");
    if (task.isLoopOverTask()) {
        // If join is part of loop over task, wait for specific iteration to get complete
        joinOn = joinOn.stream().map(name -> TaskUtils.appendIteration(name, task.getIteration())).collect(Collectors.toList());
    }
    for (String joinOnRef : joinOn) {
        Task forkedTask = workflow.getTaskByRefName(joinOnRef);
        if (forkedTask == null) {
            // Task is not even scheduled yet
            allDone = false;
            break;
        }
        Status taskStatus = forkedTask.getStatus();
        hasFailures = !taskStatus.isSuccessful() && !forkedTask.getWorkflowTask().isOptional();
        if (hasFailures) {
            failureReason.append(forkedTask.getReasonForIncompletion()).append(" ");
        }
        task.getOutputData().put(joinOnRef, forkedTask.getOutputData());
        if (!taskStatus.isTerminal()) {
            allDone = false;
        }
        if (hasFailures) {
            break;
        }
    }
    if (allDone || hasFailures) {
        if (hasFailures) {
            task.setReasonForIncompletion(failureReason.toString());
            task.setStatus(Status.FAILED);
        } else {
            task.setStatus(Status.COMPLETED);
        }
        return true;
    }
    return false;
}

Example 95 with Task

use of com.netflix.conductor.common.metadata.tasks.Task in project conductor by Netflix.

the class KafkaPublishTaskMapper method getMappedTasks.

/**
 * This method maps a {@link WorkflowTask} of type {@link TaskType#KAFKA_PUBLISH}
 * to a {@link Task} in a {@link Task.Status#SCHEDULED} state
 *
 * @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link WorkflowDef}, {@link Workflow} and a string representation of the TaskId
 * @return a List with just one Kafka task
 * @throws TerminateWorkflowException In case if the task definition does not exist
 */
@Override
public List<Task> getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException {
    logger.debug("TaskMapperContext {} in KafkaPublishTaskMapper", taskMapperContext);
    WorkflowTask taskToSchedule = taskMapperContext.getTaskToSchedule();
    Workflow workflowInstance = taskMapperContext.getWorkflowInstance();
    String taskId = taskMapperContext.getTaskId();
    int retryCount = taskMapperContext.getRetryCount();
    TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()).orElseGet(() -> Optional.ofNullable(metadataDAO.getTaskDef(taskToSchedule.getName())).orElse(null));
    Map<String, Object> input = parametersUtils.getTaskInputV2(taskToSchedule.getInputParameters(), workflowInstance, taskId, taskDefinition);
    Task kafkaPublishTask = new Task();
    kafkaPublishTask.setTaskType(taskToSchedule.getType());
    kafkaPublishTask.setTaskDefName(taskToSchedule.getName());
    kafkaPublishTask.setReferenceTaskName(taskToSchedule.getTaskReferenceName());
    kafkaPublishTask.setWorkflowInstanceId(workflowInstance.getWorkflowId());
    kafkaPublishTask.setWorkflowType(workflowInstance.getWorkflowName());
    kafkaPublishTask.setCorrelationId(workflowInstance.getCorrelationId());
    kafkaPublishTask.setScheduledTime(System.currentTimeMillis());
    kafkaPublishTask.setTaskId(taskId);
    kafkaPublishTask.setInputData(input);
    kafkaPublishTask.setStatus(Task.Status.SCHEDULED);
    kafkaPublishTask.setRetryCount(retryCount);
    kafkaPublishTask.setCallbackAfterSeconds(taskToSchedule.getStartDelay());
    kafkaPublishTask.setWorkflowTask(taskToSchedule);
    kafkaPublishTask.setWorkflowPriority(workflowInstance.getPriority());
    if (Objects.nonNull(taskDefinition)) {
        kafkaPublishTask.setExecutionNameSpace(taskDefinition.getExecutionNameSpace());
        kafkaPublishTask.setIsolationGroupId(taskDefinition.getIsolationGroupId());
        kafkaPublishTask.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency());
        kafkaPublishTask.setRateLimitFrequencyInSeconds(taskDefinition.getRateLimitFrequencyInSeconds());
    }
    return Collections.singletonList(kafkaPublishTask);
}