Examples with Workflow com.netflix.conductor.common.run.Workflow used on opensource projects

Example 76 with Workflow

use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.

the class WorkflowExecutor method decide.

/**
 * @param workflowId ID of the workflow to evaluate the state for
 * @return true if the workflow has completed (success or failed), false otherwise.
 * @throws ApplicationException If there was an error - caller should retry in this case.
 */
public boolean decide(String workflowId) {
    if (!executionLockService.acquireLock(workflowId)) {
        return false;
    }
    // If it is a new workflow, the tasks will be still empty even though include tasks is true
    Workflow workflow = executionDAOFacade.getWorkflowById(workflowId, true);
    // FIXME Backwards compatibility for legacy workflows already running.
    // This code will be removed in a future version.
    workflow = metadataMapperService.populateWorkflowWithDefinitions(workflow);
    if (workflow.getStatus().isTerminal()) {
        if (!workflow.getStatus().isSuccessful()) {
            cancelNonTerminalTasks(workflow);
        }
        return true;
    }
    try {
        DeciderService.DeciderOutcome outcome = deciderService.decide(workflow);
        if (outcome.isComplete) {
            endExecution(workflow);
            return true;
        }
        List<Task> tasksToBeScheduled = outcome.tasksToBeScheduled;
        setTaskDomains(tasksToBeScheduled, workflow);
        List<Task> tasksToBeUpdated = outcome.tasksToBeUpdated;
        boolean stateChanged = false;
        tasksToBeScheduled = dedupAndAddTasks(workflow, tasksToBeScheduled);
        for (Task task : outcome.tasksToBeScheduled) {
            if (isSystemTask.and(isNonTerminalTask).test(task)) {
                WorkflowSystemTask workflowSystemTask = WorkflowSystemTask.get(task.getTaskType());
                Workflow workflowInstance = deciderService.populateWorkflowAndTaskData(workflow);
                if (!workflowSystemTask.isAsync() && workflowSystemTask.execute(workflowInstance, task, this)) {
                    deciderService.externalizeTaskData(task);
                    tasksToBeUpdated.add(task);
                    stateChanged = true;
                }
            }
        }
        if (!outcome.tasksToBeUpdated.isEmpty() || !tasksToBeScheduled.isEmpty()) {
            executionDAOFacade.updateTasks(tasksToBeUpdated);
            executionDAOFacade.updateWorkflow(workflow);
        }
        stateChanged = scheduleTask(workflow, tasksToBeScheduled) || stateChanged;
        if (stateChanged) {
            decide(workflowId);
        }
    } catch (TerminateWorkflowException twe) {
        LOGGER.info("Execution terminated of workflow: {}", workflowId, twe);
        terminate(workflow, twe);
        return true;
    } catch (RuntimeException e) {
        LOGGER.error("Error deciding workflow: {}", workflowId, e);
        throw e;
    } finally {
        executionLockService.releaseLock(workflowId);
    }
    return false;
}

Example 77 with Workflow

use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.

the class WorkflowExecutor method skipTasksAffectedByTerminateTask.

/**
 * When a TERMINATE task runs, it only affects the workflow in which it runs; it does not do anything with
 * in-progress tasks and subworkflows that are still running. This recursive method will ensure that all tasks within
 * all subworkflows are set to SKIPPED status so they can complete.
 * @param workflow a subworkflow within the hierarchy of the original workflow containing the TERMINATE task
 */
private void skipTasksAffectedByTerminateTask(Workflow workflow) {
    if (!workflow.getStatus().isTerminal()) {
        List<Task> tasksToBeUpdated = new ArrayList<>();
        for (Task workflowTask : workflow.getTasks()) {
            if (!workflowTask.getStatus().isTerminal()) {
                workflowTask.setStatus(SKIPPED);
                tasksToBeUpdated.add(workflowTask);
            }
            if (TaskType.SUB_WORKFLOW.name().equals(workflowTask.getTaskType()) && StringUtils.isNotBlank(workflowTask.getSubWorkflowId())) {
                Workflow subWorkflow = executionDAOFacade.getWorkflowById(workflowTask.getSubWorkflowId(), true);
                if (subWorkflow != null) {
                    skipTasksAffectedByTerminateTask(subWorkflow);
                }
            }
        }
        if (!tasksToBeUpdated.isEmpty()) {
            executionDAOFacade.updateTasks(tasksToBeUpdated);
            workflow.setStatus(Workflow.WorkflowStatus.TERMINATED);
            workflow.setReasonForIncompletion("Parent workflow was terminated with a TERMINATE task");
            executionDAOFacade.updateWorkflow(workflow);
        }
    }
}

Example 78 with Workflow

use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.

the class WorkflowExecutor method rewind.

/**
 * @param workflowId           the id of the workflow to be restarted
 * @param useLatestDefinitions if true, use the latest workflow and task definitions upon restart
 * @throws ApplicationException in the following cases:
 *                              <ul>
 *                              <li>Workflow is not in a terminal state</li>
 *                              <li>Workflow definition is not found</li>
 *                              <li>Workflow is deemed non-restartable as per workflow definition</li>
 *                              </ul>
 */
public void rewind(String workflowId, boolean useLatestDefinitions) {
    Workflow workflow = executionDAOFacade.getWorkflowById(workflowId, true);
    if (!workflow.getStatus().isTerminal()) {
        String errorMsg = String.format("Workflow: %s is not in terminal state, unable to restart.", workflow);
        LOGGER.error(errorMsg);
        throw new ApplicationException(CONFLICT, errorMsg);
    }
    WorkflowDef workflowDef;
    if (useLatestDefinitions) {
        workflowDef = metadataDAO.getLatestWorkflowDef(workflow.getWorkflowName()).orElseThrow(() -> new ApplicationException(NOT_FOUND, String.format("Unable to find latest definition for %s", workflowId)));
        // setting this here to ensure backward compatibility and consistency for workflows without the embedded workflow definition
        workflow.setVersion(workflowDef.getVersion());
        workflow.setWorkflowDefinition(workflowDef);
    } else {
        workflowDef = Optional.ofNullable(workflow.getWorkflowDefinition()).orElseGet(() -> metadataDAO.getWorkflowDef(workflow.getWorkflowName(), workflow.getWorkflowVersion()).orElseThrow(() -> new ApplicationException(NOT_FOUND, String.format("Unable to find definition for %s", workflowId))));
    }
    if (!workflowDef.isRestartable() && workflow.getStatus().equals(WorkflowStatus.COMPLETED)) {
        // Can only restart non-completed workflows when the configuration is set to false
        throw new ApplicationException(CONFLICT, String.format("Workflow: %s is non-restartable", workflow));
    }
    // Reset the workflow in the primary datastore and remove from indexer; then re-create it
    executionDAOFacade.resetWorkflow(workflowId);
    workflow.getTasks().clear();
    workflow.setReasonForIncompletion(null);
    workflow.setStartTime(System.currentTimeMillis());
    workflow.setEndTime(0);
    // Change the status to running
    workflow.setStatus(WorkflowStatus.RUNNING);
    workflow.setOutput(null);
    workflow.setExternalOutputPayloadStoragePath(null);
    try {
        executionDAOFacade.createWorkflow(workflow);
    } catch (Exception e) {
        Monitors.recordWorkflowStartError(workflowDef.getName(), WorkflowContext.get().getClientApp());
        LOGGER.error("Unable to restart workflow: {}", workflowDef.getName(), e);
        terminateWorkflow(workflowId, "Error when restarting the workflow");
        throw e;
    }
    decide(workflowId);
    if (StringUtils.isNotEmpty(workflow.getParentWorkflowId())) {
        updateParentWorkflow(workflow);
        decide(workflow.getParentWorkflowId());
    }
}

Example 79 with Workflow

use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.

the class DeciderService method decide.

private DeciderOutcome decide(final Workflow workflow, List<Task> preScheduledTasks) throws TerminateWorkflowException {
    DeciderOutcome outcome = new DeciderOutcome();
    if (workflow.getStatus().isTerminal()) {
        // you cannot evaluate a terminal workflow
        LOGGER.debug("Workflow {} is already finished. Reason: {}", workflow, workflow.getReasonForIncompletion());
        return outcome;
    }
    checkWorkflowTimeout(workflow);
    if (workflow.getStatus().equals(WorkflowStatus.PAUSED)) {
        LOGGER.debug("Workflow " + workflow.getWorkflowId() + " is paused");
        return outcome;
    }
    // Filter the list of tasks and include only tasks that are not retried, not executed
    // marked to be skipped and not part of System tasks that is DECISION, FORK, JOIN
    // This list will be empty for a new workflow being started
    List<Task> pendingTasks = workflow.getTasks().stream().filter(isNonPendingTask).collect(Collectors.toList());
    // Get all the tasks that have not completed their lifecycle yet
    // This list will be empty for a new workflow
    Set<String> executedTaskRefNames = workflow.getTasks().stream().filter(Task::isExecuted).map(Task::getReferenceTaskName).collect(Collectors.toSet());
    Map<String, Task> tasksToBeScheduled = new LinkedHashMap<>();
    preScheduledTasks.forEach(preScheduledTask -> {
        tasksToBeScheduled.put(preScheduledTask.getReferenceTaskName(), preScheduledTask);
    });
    // A new workflow does not enter this code branch
    for (Task pendingTask : pendingTasks) {
        if (SystemTaskType.is(pendingTask.getTaskType()) && !pendingTask.getStatus().isTerminal()) {
            tasksToBeScheduled.putIfAbsent(pendingTask.getReferenceTaskName(), pendingTask);
            executedTaskRefNames.remove(pendingTask.getReferenceTaskName());
        }
        Optional<TaskDef> taskDefinition = pendingTask.getTaskDefinition();
        if (!taskDefinition.isPresent()) {
            taskDefinition = Optional.ofNullable(workflow.getWorkflowDefinition().getTaskByRefName(pendingTask.getReferenceTaskName())).map(WorkflowTask::getTaskDefinition);
        }
        if (taskDefinition.isPresent()) {
            checkTaskTimeout(taskDefinition.get(), pendingTask);
            checkTaskPollTimeout(taskDefinition.get(), pendingTask);
            // If the task has not been updated for "responseTimeoutSeconds" then mark task as TIMED_OUT
            if (isResponseTimedOut(taskDefinition.get(), pendingTask)) {
                timeoutTask(taskDefinition.get(), pendingTask);
            }
        }
        if (!pendingTask.getStatus().isSuccessful()) {
            WorkflowTask workflowTask = pendingTask.getWorkflowTask();
            if (workflowTask == null) {
                workflowTask = workflow.getWorkflowDefinition().getTaskByRefName(pendingTask.getReferenceTaskName());
            }
            Optional<Task> retryTask = retry(taskDefinition.orElse(null), workflowTask, pendingTask, workflow);
            if (retryTask.isPresent()) {
                tasksToBeScheduled.put(retryTask.get().getReferenceTaskName(), retryTask.get());
                executedTaskRefNames.remove(retryTask.get().getReferenceTaskName());
                outcome.tasksToBeUpdated.add(pendingTask);
            } else {
                pendingTask.setStatus(COMPLETED_WITH_ERRORS);
            }
        }
        if (!pendingTask.isExecuted() && !pendingTask.isRetried() && pendingTask.getStatus().isTerminal()) {
            pendingTask.setExecuted(true);
            List<Task> nextTasks = getNextTask(workflow, pendingTask);
            if (pendingTask.isLoopOverTask() && !TaskType.DO_WHILE.name().equals(pendingTask.getTaskType()) && !nextTasks.isEmpty()) {
                nextTasks = filterNextLoopOverTasks(nextTasks, pendingTask, workflow);
            }
            nextTasks.forEach(nextTask -> tasksToBeScheduled.putIfAbsent(nextTask.getReferenceTaskName(), nextTask));
            outcome.tasksToBeUpdated.add(pendingTask);
            LOGGER.debug("Scheduling Tasks from {}, next = {} for workflowId: {}", pendingTask.getTaskDefName(), nextTasks.stream().map(Task::getTaskDefName).collect(Collectors.toList()), workflow.getWorkflowId());
        }
    }
    // All the tasks that need to scheduled are added to the outcome, in case of
    List<Task> unScheduledTasks = tasksToBeScheduled.values().stream().filter(task -> !executedTaskRefNames.contains(task.getReferenceTaskName())).collect(Collectors.toList());
    if (!unScheduledTasks.isEmpty()) {
        LOGGER.debug("Scheduling Tasks: {} for workflow: {}", unScheduledTasks.stream().map(Task::getTaskDefName).collect(Collectors.toList()), workflow.getWorkflowId());
        outcome.tasksToBeScheduled.addAll(unScheduledTasks);
    }
    if (containsSuccessfulTerminateTask.test(workflow) || (outcome.tasksToBeScheduled.isEmpty() && checkForWorkflowCompletion(workflow))) {
        LOGGER.debug("Marking workflow: {} as complete.", workflow);
        outcome.isComplete = true;
    }
    return outcome;
}

Example 80 with Workflow

use of com.netflix.conductor.common.run.Workflow in project conductor by Netflix.

the class DeciderService method getTasksToBeScheduled.

public List<Task> getTasksToBeScheduled(Workflow workflow, WorkflowTask taskToSchedule, int retryCount, String retriedTaskId) {
    workflow = populateWorkflowAndTaskData(workflow);
    Map<String, Object> input = parametersUtils.getTaskInput(taskToSchedule.getInputParameters(), workflow, null, null);
    TaskType taskType = TaskType.USER_DEFINED;
    String type = taskToSchedule.getType();
    if (TaskType.isSystemTask(type)) {
        taskType = TaskType.valueOf(type);
    }
    // get tasks already scheduled (in progress/terminal) for  this workflow instance
    List<String> tasksInWorkflow = workflow.getTasks().stream().filter(runningTask -> runningTask.getStatus().equals(Status.IN_PROGRESS) || runningTask.getStatus().isTerminal()).map(Task::getReferenceTaskName).collect(Collectors.toList());
    String taskId = IDGenerator.generate();
    TaskMapperContext taskMapperContext = TaskMapperContext.newBuilder().withWorkflowDefinition(workflow.getWorkflowDefinition()).withWorkflowInstance(workflow).withTaskDefinition(taskToSchedule.getTaskDefinition()).withTaskToSchedule(taskToSchedule).withTaskInput(input).withRetryCount(retryCount).withRetryTaskId(retriedTaskId).withTaskId(taskId).withDeciderService(this).build();
    // for static forks, each branch of the fork creates a join task upon completion
    // for dynamic forks, a join task is created with the fork and also with each branch of the fork
    // a new task must only be scheduled if a task with the same reference name is not already in this workflow instance
    List<Task> tasks = taskMappers.get(taskType.name()).getMappedTasks(taskMapperContext).stream().filter(task -> !tasksInWorkflow.contains(task.getReferenceTaskName())).collect(Collectors.toList());
    tasks.forEach(this::externalizeTaskData);
    return tasks;
}