Examples with TaskResult com.netflix.conductor.common.metadata.tasks.TaskResult used on opensource projects

Example 51 with TaskResult

use of com.netflix.conductor.common.metadata.tasks.TaskResult in project conductor by Netflix.

the class TaskPollExecutorTest method testTaskPollException.

@Test
public void testTaskPollException() {
    Task task = testTask();
    Worker worker = mock(Worker.class);
    when(worker.getPollingInterval()).thenReturn(3000);
    when(worker.getTaskDefName()).thenReturn("test");
    when(worker.execute(any())).thenReturn(new TaskResult(task));
    TaskClient taskClient = Mockito.mock(TaskClient.class);
    when(taskClient.pollTask(any(), any(), any())).thenThrow(ConductorClientException.class).thenReturn(task);
    TaskPollExecutor taskPollExecutor = new TaskPollExecutor(null, taskClient, 1, 1, new HashMap<>(), "test-worker-");
    CountDownLatch latch = new CountDownLatch(1);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        TaskResult result = (TaskResult) args[0];
        assertEquals(IN_PROGRESS, result.getStatus());
        assertEquals(task.getTaskId(), result.getTaskId());
        latch.countDown();
        return null;
    }).when(taskClient).updateTask(any());
    Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> taskPollExecutor.pollAndExecute(worker), 0, 1, TimeUnit.SECONDS);
    Uninterruptibles.awaitUninterruptibly(latch);
    verify(taskClient).updateTask(any());
}

Example 52 with TaskResult

use of com.netflix.conductor.common.metadata.tasks.TaskResult in project conductor by Netflix.

the class TaskPollExecutor method executeTask.

private void executeTask(Worker worker, Task task) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    TaskResult result = null;
    try {
        LOGGER.debug("Executing task: {} in worker: {} at {}", task.getTaskId(), worker.getClass().getSimpleName(), worker.getIdentity());
        result = worker.execute(task);
        result.setWorkflowInstanceId(task.getWorkflowInstanceId());
        result.setTaskId(task.getTaskId());
        result.setWorkerId(worker.getIdentity());
    } catch (Exception e) {
        LOGGER.error("Unable to execute task: {} of type: {}", task.getTaskId(), task.getTaskDefName(), e);
        if (result == null) {
            task.setStatus(Task.Status.FAILED);
            result = new TaskResult(task);
        }
        handleException(e, result, worker, task);
    } finally {
        stopwatch.stop();
        MetricsContainer.getExecutionTimer(worker.getTaskDefName()).record(stopwatch.elapsed(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
    }
    LOGGER.debug("Task: {} executed by worker: {} at {} with status: {}", task.getTaskId(), worker.getClass().getSimpleName(), worker.getIdentity(), result.getStatus());
    updateWithRetry(updateRetryCount, task, result, worker);
}

Example 53 with TaskResult

use of com.netflix.conductor.common.metadata.tasks.TaskResult in project conductor by Netflix.

the class WorkflowExecutor method executeSystemTask.

// Executes the async system task
public void executeSystemTask(WorkflowSystemTask systemTask, String taskId, int callbackTime) {
    try {
        Task task = executionDAOFacade.getTaskById(taskId);
        if (task == null) {
            LOGGER.error("TaskId: {} could not be found while executing SystemTask", taskId);
            return;
        }
        LOGGER.debug("Task: {} fetched from execution DAO for taskId: {}", task, taskId);
        String queueName = QueueUtils.getQueueName(task);
        if (task.getStatus().isTerminal()) {
            // Tune the SystemTaskWorkerCoordinator's queues - if the queue size is very big this can happen!
            LOGGER.info("Task {}/{} was already completed.", task.getTaskType(), task.getTaskId());
            queueDAO.remove(queueName, task.getTaskId());
            return;
        }
        String workflowId = task.getWorkflowInstanceId();
        Workflow workflow = executionDAOFacade.getWorkflowById(workflowId, true);
        if (task.getStartTime() == 0) {
            task.setStartTime(System.currentTimeMillis());
            Monitors.recordQueueWaitTime(task.getTaskDefName(), task.getQueueWaitTime());
        }
        if (workflow.getStatus().isTerminal()) {
            LOGGER.info("Workflow {} has been completed for {}/{}", workflow.getWorkflowId(), systemTask.getName(), task.getTaskId());
            if (!task.getStatus().isTerminal()) {
                task.setStatus(CANCELED);
            }
            executionDAOFacade.updateTask(task);
            queueDAO.remove(queueName, task.getTaskId());
            return;
        }
        if (task.getStatus().equals(SCHEDULED)) {
            if (executionDAOFacade.exceedsInProgressLimit(task)) {
                // to do add a metric to record this
                LOGGER.warn("Concurrent Execution limited for {}:{}", taskId, task.getTaskDefName());
                // Postpone a message, so that it would be available for poll again.
                queueDAO.postpone(queueName, taskId, task.getWorkflowPriority(), queueTaskMessagePostponeSeconds);
                return;
            }
            if (task.getRateLimitPerFrequency() > 0 && executionDAOFacade.exceedsRateLimitPerFrequency(task, metadataDAO.getTaskDef(task.getTaskDefName()))) {
                LOGGER.warn("RateLimit Execution limited for {}:{}, limit:{}", taskId, task.getTaskDefName(), task.getRateLimitPerFrequency());
                // Postpone a message, so that it would be available for poll again.
                queueDAO.postpone(queueName, taskId, task.getWorkflowPriority(), queueTaskMessagePostponeSeconds);
                return;
            }
        }
        LOGGER.debug("Executing {}/{}-{}", task.getTaskType(), task.getTaskId(), task.getStatus());
        if (task.getStatus() == SCHEDULED || !systemTask.isAsyncComplete(task)) {
            task.setPollCount(task.getPollCount() + 1);
            executionDAOFacade.updateTask(task);
        }
        deciderService.populateTaskData(task);
        // Stop polling for asyncComplete system tasks that are not in SCHEDULED state
        if (systemTask.isAsyncComplete(task) && task.getStatus() != SCHEDULED) {
            queueDAO.remove(QueueUtils.getQueueName(task), task.getTaskId());
            return;
        }
        switch(task.getStatus()) {
            case SCHEDULED:
                systemTask.start(workflow, task, this);
                break;
            case IN_PROGRESS:
                systemTask.execute(workflow, task, this);
                break;
            default:
                break;
        }
        if (!task.getStatus().isTerminal()) {
            task.setCallbackAfterSeconds(callbackTime);
        }
        updateTask(new TaskResult(task));
        LOGGER.debug("Done Executing {}/{}-{} output={}", task.getTaskType(), task.getTaskId(), task.getStatus(), task.getOutputData().toString());
    } catch (Exception e) {
        Monitors.error(className, "executeSystemTask");
        LOGGER.error("Error executing system task - {}, with id: {}", systemTask, taskId, e);
    }
}

Example 54 with TaskResult

use of com.netflix.conductor.common.metadata.tasks.TaskResult in project orkesworkers by orkes-io.

the class ImageListMultipleConvertResizeWorker method execute.

@Override
public TaskResult execute(Task task) {
    TaskResult result = new TaskResult(task);
    try {
        List<String> fileLocations = (List<String>) task.getInputData().get("fileLocations");
        List<String> outputFormats = (List<String>) (task.getInputData().get("outputFormats"));
        List<Size> outputSizes = (List<Size>) objectMapper.convertValue(task.getInputData().get("outputSizes"), new TypeReference<List<Size>>() {
        });
        Boolean maintainAspectRatio = Boolean.valueOf((String) task.getInputData().get("maintainAspectRatio"));
        List<WorkflowTask> dynamicTasks = Lists.newArrayList();
        Map<String, Object> dynamicTasksInput = Maps.newHashMap();
        int i = 0;
        String dynamicTaskName = "image_multiple_convert_resize";
        for (String fileLocation : fileLocations) {
            String fileName = Paths.get(new URI(fileLocation).getPath()).getFileName().toString();
            String taskRefName = String.format("%s_%s_%d", dynamicTaskName, fileName, i++);
            WorkflowTask dynamicTask = new WorkflowTask();
            dynamicTask.setName(dynamicTaskName);
            dynamicTask.setTaskReferenceName(taskRefName);
            dynamicTask.setWorkflowTaskType(TaskType.SUB_WORKFLOW);
            SubWorkflowParams subWorkflowParams = new SubWorkflowParams();
            subWorkflowParams.setName("image_multiple_convert_resize");
            dynamicTask.setSubWorkflowParam(subWorkflowParams);
            dynamicTasks.add(dynamicTask);
            Map<String, Object> dynamicTaskInput = Maps.newHashMap();
            dynamicTaskInput.put("fileLocation", fileLocation);
            dynamicTaskInput.put("outputFormats", outputFormats);
            dynamicTaskInput.put("outputSizes", outputSizes);
            dynamicTaskInput.put("maintainAspectRatio", maintainAspectRatio);
            dynamicTasksInput.put(taskRefName, dynamicTaskInput);
        }
        result.setStatus(TaskResult.Status.COMPLETED);
        String currentTimeOnServer = Instant.now().toString();
        result.log("This is a test log at time: " + currentTimeOnServer);
        result.addOutputData("dynamicTasks", dynamicTasks);
        result.addOutputData("dynamicTasksInput", dynamicTasksInput);
    } catch (Exception e) {
        e.printStackTrace();
        result.setStatus(TaskResult.Status.FAILED);
        final StringWriter sw = new StringWriter();
        final PrintWriter pw = new PrintWriter(sw, true);
        e.printStackTrace(pw);
        result.log(sw.getBuffer().toString());
    }
    return result;
}

Example 55 with TaskResult

use of com.netflix.conductor.common.metadata.tasks.TaskResult in project orkesworkers by orkes-io.

the class LongRunningAsyncTaskWorker method execute.

@Override
public TaskResult execute(Task task) {
    String key = task.getWorkflowInstanceId() + "-" + task.getTaskId();
    if (taskResultsDataStore.containsKey(key)) {
        log.info("Task is already in progress - {}", key);
        return taskResultsDataStore.get(key);
    }
    TaskResult result = new TaskResult(task);
    result.setStatus(TaskResult.Status.IN_PROGRESS);
    result.setCallbackAfterSeconds(1200);
    result.getOutputData().clear();
    result.addOutputData("taskStatus", "STARTED");
    taskResultsDataStore.put(key, result);
    CompletableFuture.runAsync(() -> {
        if (task.getRetryCount() > 1 && task.getRetryCount() % 3 == 0) {
            for (int i = 0; i < 10; i++) {
                Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
                TaskResult taskResult = taskResultsDataStore.get(key);
                String outKey = "runIt-" + i;
                String value = Instant.now().toString();
                taskResult.addOutputData(outKey, value);
                taskResult.log(outKey + " - " + value);
                taskResult.setStatus(TaskResult.Status.IN_PROGRESS);
                log.info("Updating tasks offline - {}", taskResult);
                taskClient.updateTask(taskResult);
            }
            TaskResult taskResult = taskResultsDataStore.get(key);
            taskResult.addOutputData("finalUpdate", "" + Instant.now().toString());
            taskResult.setCallbackAfterSeconds(0);
            taskResult.setStatus(TaskResult.Status.COMPLETED);
            taskClient.updateTask(taskResult);
        } else {
            TaskResult taskResult = taskResultsDataStore.get(key);
            taskResult.addOutputData("failureUpdate-" + integer.incrementAndGet(), "" + Instant.now().toString());
            taskResult.setStatus(TaskResult.Status.FAILED);
            taskClient.updateTask(taskResult);
        }
    });
    log.info("Returning result from main - {}", result);
    return result;
}