use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.
the class DynamicTaskMapper method getMappedTasks.
/**
* This method maps a dynamic task to a {@link Task} based on the input params
*
* @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link WorkflowDef}, {@link Workflow} and a string representation of the TaskId
* @return A {@link List} that contains a single {@link Task} with a {@link Task.Status#SCHEDULED}
*/
@Override
public List<Task> getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException {
logger.debug("TaskMapperContext {} in DynamicTaskMapper", taskMapperContext);
WorkflowTask taskToSchedule = taskMapperContext.getTaskToSchedule();
Map<String, Object> taskInput = taskMapperContext.getTaskInput();
Workflow workflowInstance = taskMapperContext.getWorkflowInstance();
int retryCount = taskMapperContext.getRetryCount();
String retriedTaskId = taskMapperContext.getRetryTaskId();
String taskNameParam = taskToSchedule.getDynamicTaskNameParam();
String taskName = getDynamicTaskName(taskInput, taskNameParam);
taskToSchedule.setName(taskName);
TaskDef taskDefinition = getDynamicTaskDefinition(taskToSchedule);
taskToSchedule.setTaskDefinition(taskDefinition);
Map<String, Object> input = parametersUtils.getTaskInput(taskToSchedule.getInputParameters(), workflowInstance, taskDefinition, taskMapperContext.getTaskId());
Task dynamicTask = new Task();
dynamicTask.setStartDelayInSeconds(taskToSchedule.getStartDelay());
dynamicTask.setTaskId(taskMapperContext.getTaskId());
dynamicTask.setReferenceTaskName(taskToSchedule.getTaskReferenceName());
dynamicTask.setInputData(input);
dynamicTask.setWorkflowInstanceId(workflowInstance.getWorkflowId());
dynamicTask.setWorkflowType(workflowInstance.getWorkflowName());
dynamicTask.setStatus(Task.Status.SCHEDULED);
dynamicTask.setTaskType(taskToSchedule.getType());
dynamicTask.setTaskDefName(taskToSchedule.getName());
dynamicTask.setCorrelationId(workflowInstance.getCorrelationId());
dynamicTask.setScheduledTime(System.currentTimeMillis());
dynamicTask.setRetryCount(retryCount);
dynamicTask.setCallbackAfterSeconds(taskToSchedule.getStartDelay());
dynamicTask.setResponseTimeoutSeconds(taskDefinition.getResponseTimeoutSeconds());
dynamicTask.setWorkflowTask(taskToSchedule);
dynamicTask.setTaskType(taskName);
dynamicTask.setRetriedTaskId(retriedTaskId);
dynamicTask.setWorkflowPriority(workflowInstance.getPriority());
return Collections.singletonList(dynamicTask);
}
use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.
the class ForkJoinDynamicTaskMapper method getMappedTasks.
/**
* This method gets the list of tasks that need to scheduled when the task to scheduled is of type {@link TaskType#FORK_JOIN_DYNAMIC}.
* Creates a Fork Task, followed by the Dynamic tasks and a final JOIN task.
* <p>The definitions of the dynamic forks that need to be scheduled are available in the {@link WorkflowTask#getInputParameters()}
* which are accessed using the {@link TaskMapperContext#getTaskToSchedule()}. The dynamic fork task definitions are referred by a key value either by
* {@link WorkflowTask#getDynamicForkTasksParam()} or by {@link WorkflowTask#getDynamicForkJoinTasksParam()}
* </p>
* When creating the list of tasks to be scheduled a set of preconditions are validated:
* <ul>
* <li>If the input parameter representing the Dynamic fork tasks is available as part of {@link WorkflowTask#getDynamicForkTasksParam()} then
* the input for the dynamic task is validated to be a map by using {@link WorkflowTask#getDynamicForkTasksInputParamName()}</li>
* <li>If the input parameter representing the Dynamic fork tasks is available as part of {@link WorkflowTask#getDynamicForkJoinTasksParam()} then
* the input for the dynamic tasks is available in the payload of the tasks definition.
* </li>
* <li>A check is performed that the next following task in the {@link WorkflowDef} is a {@link TaskType#JOIN}</li>
* </ul>
*
* @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link WorkflowDef}, {@link Workflow} and a string representation of the TaskId
* @throws TerminateWorkflowException In case of:
* <ul>
* <li>
* When the task after {@link TaskType#FORK_JOIN_DYNAMIC} is not a {@link TaskType#JOIN}
* </li>
* <li>
* When the input parameters for the dynamic tasks are not of type {@link Map}
* </li>
* </ul>
* @return: List of tasks in the following order:
* <ul>
* <li>
* {@link SystemTaskType#FORK} with {@link Task.Status#COMPLETED}
* </li>
* <li>
* Might be any kind of task, but this is most cases is a UserDefinedTask with {@link Task.Status#SCHEDULED}
* </li>
* <li>
* {@link SystemTaskType#JOIN} with {@link Task.Status#IN_PROGRESS}
* </li>
* </ul>
*/
@Override
public List<Task> getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException {
logger.debug("TaskMapperContext {} in ForkJoinDynamicTaskMapper", taskMapperContext);
WorkflowTask taskToSchedule = taskMapperContext.getTaskToSchedule();
Workflow workflowInstance = taskMapperContext.getWorkflowInstance();
String taskId = taskMapperContext.getTaskId();
int retryCount = taskMapperContext.getRetryCount();
List<Task> mappedTasks = new LinkedList<>();
// Get the list of dynamic tasks and the input for the tasks
Pair<List<WorkflowTask>, Map<String, Map<String, Object>>> workflowTasksAndInputPair = Optional.ofNullable(taskToSchedule.getDynamicForkTasksParam()).map(dynamicForkTaskParam -> getDynamicForkTasksAndInput(taskToSchedule, workflowInstance, dynamicForkTaskParam)).orElseGet(() -> getDynamicForkJoinTasksAndInput(taskToSchedule, workflowInstance));
List<WorkflowTask> dynForkTasks = workflowTasksAndInputPair.getLeft();
Map<String, Map<String, Object>> tasksInput = workflowTasksAndInputPair.getRight();
// Create Fork Task which needs to be followed by the dynamic tasks
Task forkDynamicTask = createDynamicForkTask(taskToSchedule, workflowInstance, taskId, dynForkTasks);
mappedTasks.add(forkDynamicTask);
List<String> joinOnTaskRefs = new LinkedList<>();
// which indicates that the following task after that needs to be a join task
for (WorkflowTask dynForkTask : dynForkTasks) {
// TODO this is a cyclic dependency, break it out using function composition
List<Task> forkedTasks = taskMapperContext.getDeciderService().getTasksToBeScheduled(workflowInstance, dynForkTask, retryCount);
// same reference name has already been created in the Workflow.
if (forkedTasks == null || forkedTasks.isEmpty()) {
Optional<String> existingTaskRefName = workflowInstance.getTasks().stream().filter(runningTask -> runningTask.getStatus().equals(Task.Status.IN_PROGRESS) || runningTask.getStatus().isTerminal()).map(Task::getReferenceTaskName).filter(refTaskName -> refTaskName.equals(dynForkTask.getTaskReferenceName())).findAny();
// Construct an informative error message
String terminateMessage = "No dynamic tasks could be created for the Workflow: " + workflowInstance.toShortString() + ", Dynamic Fork Task: " + dynForkTask;
if (existingTaskRefName.isPresent()) {
terminateMessage += "Attempted to create a duplicate task reference name: " + existingTaskRefName.get();
}
throw new TerminateWorkflowException(terminateMessage);
}
for (Task forkedTask : forkedTasks) {
Map<String, Object> forkedTaskInput = tasksInput.get(forkedTask.getReferenceTaskName());
forkedTask.getInputData().putAll(forkedTaskInput);
}
mappedTasks.addAll(forkedTasks);
// Get the last of the dynamic tasks so that the join can be performed once this task is done
Task last = forkedTasks.get(forkedTasks.size() - 1);
joinOnTaskRefs.add(last.getReferenceTaskName());
}
// From the workflow definition get the next task and make sure that it is a JOIN task.
// The dynamic fork tasks need to be followed by a join task
WorkflowTask joinWorkflowTask = workflowInstance.getWorkflowDefinition().getNextTask(taskToSchedule.getTaskReferenceName());
if (joinWorkflowTask == null || !joinWorkflowTask.getType().equals(TaskType.JOIN.name())) {
throw new TerminateWorkflowException("Dynamic join definition is not followed by a join task. Check the blueprint");
}
// Create Join task
HashMap<String, Object> joinInput = new HashMap<>();
joinInput.put("joinOn", joinOnTaskRefs);
Task joinTask = createJoinTask(workflowInstance, joinWorkflowTask, joinInput);
mappedTasks.add(joinTask);
return mappedTasks;
}
use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.
the class DoWhile method getEvaluatedCondition.
@VisibleForTesting
boolean getEvaluatedCondition(Workflow workflow, Task task, WorkflowExecutor workflowExecutor) throws ScriptException {
TaskDef taskDefinition = null;
try {
taskDefinition = workflowExecutor.getTaskDefinition(task);
} catch (TerminateWorkflowException e) {
// It is ok to not have a task definition for a DO_WHILE task
}
Map<String, Object> taskInput = parametersUtils.getTaskInputV2(task.getWorkflowTask().getInputParameters(), workflow, task.getTaskId(), taskDefinition);
taskInput.put(task.getReferenceTaskName(), task.getOutputData());
List<Task> loopOver = workflow.getTasks().stream().filter(t -> (task.getWorkflowTask().has(TaskUtils.removeIterationFromTaskRefName(t.getReferenceTaskName())) && !task.getReferenceTaskName().equals(t.getReferenceTaskName()))).collect(Collectors.toList());
for (Task loopOverTask : loopOver) {
taskInput.put(TaskUtils.removeIterationFromTaskRefName(loopOverTask.getReferenceTaskName()), loopOverTask.getOutputData());
}
String condition = task.getWorkflowTask().getLoopCondition();
boolean shouldContinue = false;
if (condition != null) {
logger.debug("Condition: {} is being evaluated", condition);
// Evaluate the expression by using the Nashhorn based script evaluator
shouldContinue = ScriptEvaluator.evalBool(condition, taskInput);
}
return shouldContinue;
}
use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.
the class KafkaPublishTaskMapper method getMappedTasks.
/**
* This method maps a {@link WorkflowTask} of type {@link TaskType#KAFKA_PUBLISH}
* to a {@link Task} in a {@link Task.Status#SCHEDULED} state
*
* @param taskMapperContext: A wrapper class containing the {@link WorkflowTask}, {@link WorkflowDef}, {@link Workflow} and a string representation of the TaskId
* @return a List with just one Kafka task
* @throws TerminateWorkflowException In case if the task definition does not exist
*/
@Override
public List<Task> getMappedTasks(TaskMapperContext taskMapperContext) throws TerminateWorkflowException {
logger.debug("TaskMapperContext {} in KafkaPublishTaskMapper", taskMapperContext);
WorkflowTask taskToSchedule = taskMapperContext.getTaskToSchedule();
Workflow workflowInstance = taskMapperContext.getWorkflowInstance();
String taskId = taskMapperContext.getTaskId();
int retryCount = taskMapperContext.getRetryCount();
TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()).orElseGet(() -> Optional.ofNullable(metadataDAO.getTaskDef(taskToSchedule.getName())).orElse(null));
Map<String, Object> input = parametersUtils.getTaskInputV2(taskToSchedule.getInputParameters(), workflowInstance, taskId, taskDefinition);
Task kafkaPublishTask = new Task();
kafkaPublishTask.setTaskType(taskToSchedule.getType());
kafkaPublishTask.setTaskDefName(taskToSchedule.getName());
kafkaPublishTask.setReferenceTaskName(taskToSchedule.getTaskReferenceName());
kafkaPublishTask.setWorkflowInstanceId(workflowInstance.getWorkflowId());
kafkaPublishTask.setWorkflowType(workflowInstance.getWorkflowName());
kafkaPublishTask.setCorrelationId(workflowInstance.getCorrelationId());
kafkaPublishTask.setScheduledTime(System.currentTimeMillis());
kafkaPublishTask.setTaskId(taskId);
kafkaPublishTask.setInputData(input);
kafkaPublishTask.setStatus(Task.Status.SCHEDULED);
kafkaPublishTask.setRetryCount(retryCount);
kafkaPublishTask.setCallbackAfterSeconds(taskToSchedule.getStartDelay());
kafkaPublishTask.setWorkflowTask(taskToSchedule);
kafkaPublishTask.setWorkflowPriority(workflowInstance.getPriority());
if (Objects.nonNull(taskDefinition)) {
kafkaPublishTask.setExecutionNameSpace(taskDefinition.getExecutionNameSpace());
kafkaPublishTask.setIsolationGroupId(taskDefinition.getIsolationGroupId());
kafkaPublishTask.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency());
kafkaPublishTask.setRateLimitFrequencyInSeconds(taskDefinition.getRateLimitFrequencyInSeconds());
}
return Collections.singletonList(kafkaPublishTask);
}
use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.
the class LambdaTaskMapper method getMappedTasks.
@Override
public List<Task> getMappedTasks(TaskMapperContext taskMapperContext) {
LOGGER.debug("TaskMapperContext {} in LambdaTaskMapper", taskMapperContext);
WorkflowTask taskToSchedule = taskMapperContext.getTaskToSchedule();
Workflow workflowInstance = taskMapperContext.getWorkflowInstance();
String taskId = taskMapperContext.getTaskId();
TaskDef taskDefinition = Optional.ofNullable(taskMapperContext.getTaskDefinition()).orElseGet(() -> Optional.ofNullable(metadataDAO.getTaskDef(taskToSchedule.getName())).orElse(null));
Map<String, Object> taskInput = parametersUtils.getTaskInputV2(taskMapperContext.getTaskToSchedule().getInputParameters(), workflowInstance, taskId, taskDefinition);
Task lambdaTask = new Task();
lambdaTask.setTaskType(Lambda.TASK_NAME);
lambdaTask.setTaskDefName(taskMapperContext.getTaskToSchedule().getName());
lambdaTask.setReferenceTaskName(taskMapperContext.getTaskToSchedule().getTaskReferenceName());
lambdaTask.setWorkflowInstanceId(workflowInstance.getWorkflowId());
lambdaTask.setWorkflowType(workflowInstance.getWorkflowName());
lambdaTask.setCorrelationId(workflowInstance.getCorrelationId());
lambdaTask.setStartTime(System.currentTimeMillis());
lambdaTask.setScheduledTime(System.currentTimeMillis());
lambdaTask.setInputData(taskInput);
lambdaTask.setTaskId(taskId);
lambdaTask.setStatus(Task.Status.IN_PROGRESS);
lambdaTask.setWorkflowTask(taskToSchedule);
lambdaTask.setWorkflowPriority(workflowInstance.getPriority());
return Collections.singletonList(lambdaTask);
}
Aggregations