Examples with WorkflowSystemTask com.netflix.conductor.core.execution.tasks.WorkflowSystemTask used on opensource projects

Example 6 with WorkflowSystemTask

use of com.netflix.conductor.core.execution.tasks.WorkflowSystemTask in project conductor by Netflix.

the class TestWorkflowRepairService method assertSyncSystemTasksAreNotCheckedAgainstQueue.

@Test
public void assertSyncSystemTasksAreNotCheckedAgainstQueue() {
    // Create a Decision object to init WorkflowSystemTask registry.
    Decision decision = new Decision();
    Task task = new Task();
    task.setTaskType("DECISION");
    task.setStatus(Task.Status.SCHEDULED);
    assertFalse(workflowRepairService.verifyAndRepairTask(task));
    // Verify that queue contains is never checked for sync system tasks
    verify(queueDAO, never()).containsMessage(anyString(), anyString());
    // Verify that queue message is never pushed for sync system tasks
    verify(queueDAO, never()).push(anyString(), anyString(), anyLong());
}

Example 7 with WorkflowSystemTask

use of com.netflix.conductor.core.execution.tasks.WorkflowSystemTask in project conductor by Netflix.

the class WorkflowExecutor method executeSystemTask.

// Executes the async system task
public void executeSystemTask(WorkflowSystemTask systemTask, String taskId, int callbackTime) {
    try {
        Task task = executionDAOFacade.getTaskById(taskId);
        if (task == null) {
            LOGGER.error("TaskId: {} could not be found while executing SystemTask", taskId);
            return;
        }
        LOGGER.debug("Task: {} fetched from execution DAO for taskId: {}", task, taskId);
        String queueName = QueueUtils.getQueueName(task);
        if (task.getStatus().isTerminal()) {
            // Tune the SystemTaskWorkerCoordinator's queues - if the queue size is very big this can happen!
            LOGGER.info("Task {}/{} was already completed.", task.getTaskType(), task.getTaskId());
            queueDAO.remove(queueName, task.getTaskId());
            return;
        }
        String workflowId = task.getWorkflowInstanceId();
        Workflow workflow = executionDAOFacade.getWorkflowById(workflowId, true);
        if (task.getStartTime() == 0) {
            task.setStartTime(System.currentTimeMillis());
            Monitors.recordQueueWaitTime(task.getTaskDefName(), task.getQueueWaitTime());
        }
        if (workflow.getStatus().isTerminal()) {
            LOGGER.info("Workflow {} has been completed for {}/{}", workflow.getWorkflowId(), systemTask.getName(), task.getTaskId());
            if (!task.getStatus().isTerminal()) {
                task.setStatus(CANCELED);
            }
            executionDAOFacade.updateTask(task);
            queueDAO.remove(queueName, task.getTaskId());
            return;
        }
        if (task.getStatus().equals(SCHEDULED)) {
            if (executionDAOFacade.exceedsInProgressLimit(task)) {
                // to do add a metric to record this
                LOGGER.warn("Concurrent Execution limited for {}:{}", taskId, task.getTaskDefName());
                // Postpone a message, so that it would be available for poll again.
                queueDAO.postpone(queueName, taskId, task.getWorkflowPriority(), queueTaskMessagePostponeSeconds);
                return;
            }
            if (task.getRateLimitPerFrequency() > 0 && executionDAOFacade.exceedsRateLimitPerFrequency(task, metadataDAO.getTaskDef(task.getTaskDefName()))) {
                LOGGER.warn("RateLimit Execution limited for {}:{}, limit:{}", taskId, task.getTaskDefName(), task.getRateLimitPerFrequency());
                // Postpone a message, so that it would be available for poll again.
                queueDAO.postpone(queueName, taskId, task.getWorkflowPriority(), queueTaskMessagePostponeSeconds);
                return;
            }
        }
        LOGGER.debug("Executing {}/{}-{}", task.getTaskType(), task.getTaskId(), task.getStatus());
        if (task.getStatus() == SCHEDULED || !systemTask.isAsyncComplete(task)) {
            task.setPollCount(task.getPollCount() + 1);
            executionDAOFacade.updateTask(task);
        }
        deciderService.populateTaskData(task);
        // Stop polling for asyncComplete system tasks that are not in SCHEDULED state
        if (systemTask.isAsyncComplete(task) && task.getStatus() != SCHEDULED) {
            queueDAO.remove(QueueUtils.getQueueName(task), task.getTaskId());
            return;
        }
        switch(task.getStatus()) {
            case SCHEDULED:
                systemTask.start(workflow, task, this);
                break;
            case IN_PROGRESS:
                systemTask.execute(workflow, task, this);
                break;
            default:
                break;
        }
        if (!task.getStatus().isTerminal()) {
            task.setCallbackAfterSeconds(callbackTime);
        }
        updateTask(new TaskResult(task));
        LOGGER.debug("Done Executing {}/{}-{} output={}", task.getTaskType(), task.getTaskId(), task.getStatus(), task.getOutputData().toString());
    } catch (Exception e) {
        Monitors.error(className, "executeSystemTask");
        LOGGER.error("Error executing system task - {}, with id: {}", systemTask, taskId, e);
    }
}

Example 8 with WorkflowSystemTask

use of com.netflix.conductor.core.execution.tasks.WorkflowSystemTask in project conductor by Netflix.

the class WorkflowExecutor method scheduleTask.

@VisibleForTesting
boolean scheduleTask(Workflow workflow, List<Task> tasks) {
    List<Task> createdTasks;
    List<Task> tasksToBeQueued;
    boolean startedSystemTasks = false;
    try {
        if (tasks == null || tasks.isEmpty()) {
            return false;
        }
        // Get the highest seq number
        int count = workflow.getTasks().stream().mapToInt(Task::getSeq).max().orElse(0);
        for (Task task : tasks) {
            if (task.getSeq() == 0) {
                // Set only if the seq was not set
                task.setSeq(++count);
            }
        }
        // metric to track the distribution of number of tasks within a workflow
        Monitors.recordNumTasksInWorkflow(workflow.getTasks().size() + tasks.size(), workflow.getWorkflowName(), String.valueOf(workflow.getWorkflowVersion()));
        // Save the tasks in the DAO
        createdTasks = executionDAOFacade.createTasks(tasks);
        List<Task> systemTasks = createdTasks.stream().filter(isSystemTask).collect(Collectors.toList());
        tasksToBeQueued = createdTasks.stream().filter(isSystemTask.negate()).collect(Collectors.toList());
        // Traverse through all the system tasks, start the sync tasks, in case of async queue the tasks
        for (Task task : systemTasks) {
            WorkflowSystemTask workflowSystemTask = WorkflowSystemTask.get(task.getTaskType());
            if (workflowSystemTask == null) {
                throw new ApplicationException(NOT_FOUND, "No system task found by name " + task.getTaskType());
            }
            if (task.getStatus() != null && !task.getStatus().isTerminal() && task.getStartTime() == 0) {
                task.setStartTime(System.currentTimeMillis());
            }
            if (!workflowSystemTask.isAsync()) {
                try {
                    deciderService.populateTaskData(task);
                    workflowSystemTask.start(workflow, task, this);
                } catch (Exception e) {
                    String errorMsg = String.format("Unable to start system task: %s, {id: %s, name: %s}", task.getTaskType(), task.getTaskId(), task.getTaskDefName());
                    throw new ApplicationException(Code.INTERNAL_ERROR, errorMsg, e);
                }
                startedSystemTasks = true;
                deciderService.externalizeTaskData(task);
                executionDAOFacade.updateTask(task);
            } else {
                tasksToBeQueued.add(task);
            }
        }
    } catch (Exception e) {
        List<String> taskIds = tasks.stream().map(Task::getTaskId).collect(Collectors.toList());
        String errorMsg = String.format("Error scheduling tasks: %s, for workflow: %s", taskIds, workflow.getWorkflowId());
        LOGGER.error(errorMsg, e);
        Monitors.error(className, "scheduleTask");
        // rollbackTasks(workflow.getWorkflowId(), createdTasks);
        throw new TerminateWorkflowException(errorMsg);
    }
    // On addTaskToQueue failures, ignore the exceptions and let WorkflowRepairService take care of republishing the messages to the queue.
    try {
        addTaskToQueue(tasksToBeQueued);
    } catch (Exception e) {
        List<String> taskIds = tasksToBeQueued.stream().map(Task::getTaskId).collect(Collectors.toList());
        String errorMsg = String.format("Error pushing tasks to the queue: %s, for workflow: %s", taskIds, workflow.getWorkflowId());
        LOGGER.warn(errorMsg, e);
        Monitors.error(className, "scheduleTask");
    }
    return startedSystemTasks;
}

Example 9 with WorkflowSystemTask

use of com.netflix.conductor.core.execution.tasks.WorkflowSystemTask in project conductor by Netflix.

the class WorkflowExecutor method cancelNonTerminalTasks.

@VisibleForTesting
List<String> cancelNonTerminalTasks(Workflow workflow) {
    List<String> erroredTasks = new ArrayList<>();
    // Update non-terminal tasks' status to CANCELED
    for (Task task : workflow.getTasks()) {
        if (!task.getStatus().isTerminal()) {
            // Cancel the ones which are not completed yet....
            task.setStatus(CANCELED);
            if (isSystemTask.test(task)) {
                WorkflowSystemTask workflowSystemTask = WorkflowSystemTask.get(task.getTaskType());
                try {
                    workflowSystemTask.cancel(workflow, task, this);
                } catch (Exception e) {
                    erroredTasks.add(task.getReferenceTaskName());
                    LOGGER.error("Error canceling system task:{}/{} in workflow: {}", workflowSystemTask.getName(), task.getTaskId(), workflow.getWorkflowId(), e);
                }
            }
            executionDAOFacade.updateTask(task);
        }
    }
    if (erroredTasks.isEmpty()) {
        try {
            workflowStatusListener.onWorkflowFinalizedIfEnabled(workflow);
            queueDAO.remove(DECIDER_QUEUE, workflow.getWorkflowId());
        } catch (Exception e) {
            LOGGER.error("Error removing workflow: {} from decider queue", workflow.getWorkflowId(), e);
        }
    }
    return erroredTasks;
}