Examples with TaskRun io.kestra.core.models.executions.TaskRun used on opensource projects

Example 6 with TaskRun

use of io.kestra.core.models.executions.TaskRun in project kestra by kestra-io.

the class GraphService method fillGraph.

private static void fillGraph(GraphCluster graph, List<Task> tasks, RelationType relationType, TaskRun parent, Execution execution, String value) throws IllegalVariableEvaluationException {
    Iterator<Task> iterator = tasks.iterator();
    AbstractGraphTask previous = graph.getRoot();
    boolean isFirst = true;
    while (iterator.hasNext()) {
        Task currentTask = iterator.next();
        for (TaskRun currentTaskRun : findTaskRuns(currentTask, execution, parent)) {
            AbstractGraphTask currentGraph;
            List<String> parentValues = null;
            // we use the graph relation type by default but we change it to pass relation for case below
            RelationType newRelation = graph.getRelationType();
            if (relationType == RelationType.ERROR) {
                newRelation = relationType;
            } else if ((!isFirst && relationType != RelationType.PARALLEL && graph.getRelationType() != RelationType.DYNAMIC)) {
                newRelation = relationType;
            }
            Relation relation = new Relation(newRelation, currentTaskRun == null ? value : currentTaskRun.getValue());
            if (execution != null && currentTaskRun != null) {
                parentValues = execution.findChildsValues(currentTaskRun, true);
            }
            // detect kind
            if (currentTask instanceof FlowableTask) {
                FlowableTask<?> flowableTask = ((FlowableTask<?>) currentTask);
                currentGraph = flowableTask.tasksTree(execution, currentTaskRun, parentValues);
            } else {
                currentGraph = new GraphTask(currentTask, currentTaskRun, parentValues, relationType);
            }
            // add the node
            graph.getGraph().addNode(currentGraph);
            // link to previous one
            if (previous != null) {
                graph.getGraph().addEdge(previous instanceof GraphCluster ? ((GraphCluster) previous).getEnd() : previous, currentGraph instanceof GraphCluster ? ((GraphCluster) currentGraph).getRoot() : currentGraph, relation);
            }
            // change previous for current one to link
            if (relationType != RelationType.PARALLEL) {
                previous = currentGraph;
            }
            // link to end task
            if (GraphService.isAllLinkToEnd(relationType)) {
                graph.getGraph().addEdge(currentGraph instanceof GraphCluster ? ((GraphCluster) currentGraph).getEnd() : currentGraph, graph.getEnd(), new Relation());
            }
            isFirst = false;
            if (!iterator.hasNext() && !isAllLinkToEnd(relationType)) {
                graph.getGraph().addEdge(currentGraph instanceof GraphCluster ? ((GraphCluster) currentGraph).getEnd() : currentGraph, graph.getEnd(), new Relation());
            }
        }
    }
}

Example 7 with TaskRun

use of io.kestra.core.models.executions.TaskRun in project kestra by kestra-io.

the class Worker method run.

private void run(WorkerTask workerTask) throws QueueException {
    metricRegistry.counter(MetricRegistry.METRIC_WORKER_STARTED_COUNT, metricRegistry.tags(workerTask)).increment();
    if (workerTask.getTaskRun().getState().getCurrent() == State.Type.CREATED) {
        metricRegistry.timer(MetricRegistry.METRIC_WORKER_QUEUED_DURATION, metricRegistry.tags(workerTask)).record(Duration.between(workerTask.getTaskRun().getState().getStartDate(), now()));
    }
    workerTask.logger().info("[namespace: {}] [flow: {}] [task: {}] [execution: {}] [taskrun: {}] [value: {}] Type {} started", workerTask.getTaskRun().getNamespace(), workerTask.getTaskRun().getFlowId(), workerTask.getTaskRun().getTaskId(), workerTask.getTaskRun().getExecutionId(), workerTask.getTaskRun().getId(), workerTask.getTaskRun().getValue(), workerTask.getTask().getClass().getSimpleName());
    if (workerTask.logger().isDebugEnabled()) {
        workerTask.logger().debug("Variables\n{}", JacksonMapper.log(workerTask.getRunContext().getVariables()));
    }
    workerTask = workerTask.withTaskRun(workerTask.getTaskRun().withState(State.Type.RUNNING));
    this.workerTaskResultQueue.emit(new WorkerTaskResult(workerTask));
    if (workerTask.getTask() instanceof RunnableTask) {
        // killed cased
        if (killedExecution.contains(workerTask.getTaskRun().getExecutionId())) {
            workerTask = workerTask.withTaskRun(workerTask.getTaskRun().withState(State.Type.KILLED));
            this.workerTaskResultQueue.emit(new WorkerTaskResult(workerTask));
            this.logTerminated(workerTask);
            return;
        }
        AtomicReference<WorkerTask> current = new AtomicReference<>(workerTask);
        // run
        WorkerTask finalWorkerTask = Failsafe.with(AbstractRetry.<WorkerTask>retryPolicy(workerTask.getTask().getRetry()).handleResultIf(result -> result.getTaskRun().lastAttempt() != null && Objects.requireNonNull(result.getTaskRun().lastAttempt()).getState().getCurrent() == State.Type.FAILED).onRetry(e -> {
            WorkerTask lastResult = e.getLastResult();
            lastResult = lastResult.getRunContext().cleanup(lastResult);
            current.set(lastResult);
            metricRegistry.counter(MetricRegistry.METRIC_WORKER_RETRYED_COUNT, metricRegistry.tags(current.get(), MetricRegistry.TAG_ATTEMPT_COUNT, String.valueOf(e.getAttemptCount()))).increment();
            this.workerTaskResultQueue.emit(new WorkerTaskResult(lastResult));
        })).get(() -> this.runAttempt(current.get()));
        finalWorkerTask = finalWorkerTask.getRunContext().cleanup(finalWorkerTask);
        // get last state
        TaskRunAttempt lastAttempt = finalWorkerTask.getTaskRun().lastAttempt();
        if (lastAttempt == null) {
            throw new IllegalStateException("Can find lastAttempt on taskRun '" + finalWorkerTask.getTaskRun().toString(true) + "'");
        }
        State.Type state = lastAttempt.getState().getCurrent();
        if (workerTask.getTask().getRetry() != null && workerTask.getTask().getRetry().getWarningOnRetry() && finalWorkerTask.getTaskRun().getAttempts().size() > 0 && state == State.Type.SUCCESS) {
            state = State.Type.WARNING;
        }
        // emit
        finalWorkerTask = finalWorkerTask.withTaskRun(finalWorkerTask.getTaskRun().withState(state));
        // changing status must work in order to finish current task (except if we are near the upper bound size).
        try {
            this.workerTaskResultQueue.emit(new WorkerTaskResult(finalWorkerTask));
        } catch (QueueException e) {
            finalWorkerTask = workerTask.withTaskRun(workerTask.getTaskRun().withState(State.Type.FAILED));
            this.workerTaskResultQueue.emit(new WorkerTaskResult(finalWorkerTask));
        } finally {
            this.logTerminated(finalWorkerTask);
        }
    }
}

Example 8 with TaskRun

use of io.kestra.core.models.executions.TaskRun in project kestra by kestra-io.

the class ExecutionService method replay.

public Execution replay(final Execution execution, String taskRunId, @Nullable Integer revision) throws Exception {
    if (!execution.getState().isTerninated()) {
        throw new IllegalStateException("Execution must be terminated to be restarted, " + "current state is '" + execution.getState().getCurrent() + "' !");
    }
    final Flow flow = flowRepositoryInterface.findByExecution(execution);
    GraphCluster graphCluster = GraphService.of(flow, execution);
    Set<String> taskRunToRestart = this.taskRunWithAncestors(execution, execution.getTaskRunList().stream().filter(taskRun -> taskRun.getId().equals(taskRunId)).collect(Collectors.toList()));
    if (taskRunToRestart.size() == 0) {
        throw new IllegalArgumentException("No task found to restart execution from !");
    }
    Map<String, String> mappingTaskRunId = this.mapTaskRunId(execution, false);
    final String newExecutionId = IdUtils.create();
    List<TaskRun> newTaskRuns = execution.getTaskRunList().stream().map(throwFunction(originalTaskRun -> this.mapTaskRun(flow, originalTaskRun, mappingTaskRunId, newExecutionId, State.Type.RESTARTED, taskRunToRestart.contains(originalTaskRun.getId())))).collect(Collectors.toList());
    Set<String> taskRunToRemove = GraphService.successors(graphCluster, List.of(taskRunId)).stream().filter(task -> task.getTaskRun() != null).filter(task -> !task.getTaskRun().getId().equals(taskRunId)).filter(task -> !taskRunToRestart.contains(task.getTaskRun().getId())).map(s -> mappingTaskRunId.get(s.getTaskRun().getId())).collect(Collectors.toSet());
    taskRunToRemove.forEach(r -> newTaskRuns.removeIf(taskRun -> taskRun.getId().equals(r)));
    // Build and launch new execution
    Execution newExecution = execution.childExecution(newExecutionId, newTaskRuns, execution.withState(State.Type.RESTARTED).getState());
    return revision != null ? newExecution.withFlowRevision(revision) : newExecution;
}

Example 9 with TaskRun

use of io.kestra.core.models.executions.TaskRun in project kestra by kestra-io.

the class ExecutorService method childWorkerTaskResult.

private Optional<WorkerTaskResult> childWorkerTaskResult(Flow flow, Execution execution, TaskRun parentTaskRun) throws InternalException {
    Task parent = flow.findTaskByTaskId(parentTaskRun.getTaskId());
    if (parent instanceof FlowableTask) {
        FlowableTask<?> flowableParent = (FlowableTask<?>) parent;
        RunContext runContext = runContextFactory.of(flow, parent, execution, parentTaskRun);
        // first find the normal ended child tasks and send result
        Optional<WorkerTaskResult> endedTask = childWorkerTaskTypeToWorkerTask(flowableParent.resolveState(runContext, execution, parentTaskRun), parent, parentTaskRun);
        if (endedTask.isPresent()) {
            return endedTask;
        }
        // after if the execution is KILLING, we find if all already started tasks if finished
        if (execution.getState().getCurrent() == State.Type.KILLING) {
            // first notified the parent taskRun of killing to avoid new creation of tasks
            if (parentTaskRun.getState().getCurrent() != State.Type.KILLING) {
                return childWorkerTaskTypeToWorkerTask(Optional.of(State.Type.KILLING), parent, parentTaskRun);
            }
            // Then wait for completion (KILLED or whatever) on child taks to KILLED the parennt one.
            List<ResolvedTask> currentTasks = execution.findTaskDependingFlowState(flowableParent.childTasks(runContext, parentTaskRun), FlowableUtils.resolveTasks(flowableParent.getErrors(), parentTaskRun));
            List<TaskRun> taskRunByTasks = execution.findTaskRunByTasks(currentTasks, parentTaskRun);
            if (taskRunByTasks.stream().filter(t -> t.getState().isTerninated()).count() == taskRunByTasks.size()) {
                return childWorkerTaskTypeToWorkerTask(Optional.of(State.Type.KILLED), parent, parentTaskRun);
            }
        }
    }
    return Optional.empty();
}

Example 10 with TaskRun

use of io.kestra.core.models.executions.TaskRun in project kestra by kestra-io.

the class ExecutorService method handleChildNext.

private Executor handleChildNext(Executor executor) throws InternalException {
    if (executor.getExecution().getTaskRunList() == null) {
        return executor;
    }
    List<TaskRun> running = executor.getExecution().getTaskRunList().stream().filter(taskRun -> taskRun.getState().isRunning()).collect(Collectors.toList());
    // Remove functionnal style to avoid (class io.kestra.core.exceptions.IllegalVariableEvaluationException cannot be cast to class java.lang.RuntimeException'
    ArrayList<TaskRun> result = new ArrayList<>();
    for (TaskRun taskRun : running) {
        result.addAll(this.childNextsTaskRun(executor, taskRun));
    }
    if (result.size() == 0) {
        return executor;
    }
    return executor.withTaskRun(result, "handleChildNext");
}