Examples with Executor io.kestra.core.runners.Executor used on opensource projects

Example 1 with Executor

use of io.kestra.core.runners.Executor in project kestra by kestra-io.

the class ExecutorTriggerCleaner method topology.

public StreamsBuilder topology() {
    StreamsBuilder builder = new KafkaStreamsBuilder();
    KStream<String, Executor> executorKStream = kafkaStreamSourceService.executorKStream(builder);
    KStream<String, Executor> executionWithFlowKStream = kafkaStreamSourceService.executorWithFlow(executorKStream, false);
    GlobalKTable<String, Trigger> triggerGlobalKTable = kafkaStreamSourceService.triggerGlobalKTable(builder);
    executionWithFlowKStream.filter((key, value) -> value.getExecution().getTrigger() != null, Named.as("cleanTrigger-hasTrigger-filter")).filter((key, value) -> conditionService.isTerminatedWithListeners(value.getFlow(), value.getExecution()), Named.as("cleanTrigger-terminated-filter")).join(triggerGlobalKTable, (key, executionWithFlow) -> Trigger.uid(executionWithFlow.getExecution()), (execution, trigger) -> trigger.resetExecution(), Named.as("cleanTrigger-join")).selectKey((key, value) -> queueService.key(value)).to(kafkaAdminService.getTopicName(Trigger.class), Produced.with(Serdes.String(), JsonSerde.of(Trigger.class)));
    return builder;
}

Example 2 with Executor

use of io.kestra.core.runners.Executor in project kestra by kestra-io.

the class ExecutorFromExecutionTransformer method transform.

@Override
public Executor transform(final String key, final Execution value) {
    if (value == null) {
        return null;
    }
    Executor executor = new Executor(value, this.context.offset());
    // restart need to be saved on state store for future join
    if (executor.getExecution().getState().getCurrent() == State.Type.RESTARTED) {
        store.put(key, executor.serialize());
    }
    this.context.headers().remove("from");
    this.context.headers().remove("offset");
    return executor;
}

Example 3 with Executor

use of io.kestra.core.runners.Executor in project kestra by kestra-io.

the class ExecutorNextTransformer method transform.

@Override
public Executor transform(final String key, final Executor value) {
    Executor executor = executorService.process(value);
    if (executor.getNexts().size() == 0) {
        return value;
    }
    Store store = this.store.get(key) == null ? new Store() : this.store.get(key);
    Map<Boolean, List<String>> groups = executor.getNexts().stream().map(taskRun -> taskRun.getParentTaskRunId() + "-" + taskRun.getTaskId() + "-" + taskRun.getValue()).collect(Collectors.partitioningBy(store::contains));
    if (groups.get(true).size() > 0) {
        groups.get(true).forEach(s -> log.trace("Duplicate next taskRun for execution '{}', value '{}'", key, s));
        return value;
    }
    store.addAll(groups.get(false));
    this.store.put(key, store);
    Execution newExecution = executorService.onNexts(value.getFlow(), value.getExecution(), executor.getNexts());
    return value.withExecution(newExecution, "onNexts");
}

Example 4 with Executor

use of io.kestra.core.runners.Executor in project kestra by kestra-io.

the class ExecutorJoinerTransformer method transform.

@Override
public Executor transform(final String key, final Executor value) {
    if (value.getExecution() != null) {
        return value;
    }
    WorkerTaskResult workerTaskResult = value.getJoined();
    if (log.isDebugEnabled()) {
        log.debug("<< IN  WorkerTaskResult [key='{}', partition='{}, offset='{}'] : {}", key, context.partition(), context.offset(), workerTaskResult.getTaskRun().toStringState());
    }
    Executor executor = this.store.get(key);
    // already purge execution ?
    if (executor == null) {
        log.warn("Unable to find Executor with key '" + key + "' for WorkerTaskResult id '" + workerTaskResult.getTaskRun().getId() + "' '" + workerTaskResult.getTaskRun().toStringState() + "'");
        return null;
    }
    if (!executor.getExecution().hasTaskRunJoinable(value.getJoined().getTaskRun())) {
        return executor;
    }
    try {
        Execution newExecution = executor.getExecution().withTaskRun(workerTaskResult.getTaskRun());
        executor = executor.withExecution(newExecution, "joinWorkerResult");
    } catch (Exception e) {
        return executor.withException(e, "joinWorkerResult");
    }
    // send metrics
    metricRegistry.counter(MetricRegistry.KESTRA_EXECUTOR_TASKRUN_ENDED_COUNT, metricRegistry.tags(workerTaskResult)).increment();
    metricRegistry.timer(MetricRegistry.KESTRA_EXECUTOR_TASKRUN_ENDED_DURATION, metricRegistry.tags(workerTaskResult)).record(workerTaskResult.getTaskRun().getState().getDuration());
    return executor;
}

Example 5 with Executor

use of io.kestra.core.runners.Executor in project kestra by kestra-io.

the class FlowJoinerTransformer method transform.

@Override
public Executor transform(String key, Executor executor) {
    Flow flow;
    try {
        // pooling of new flow can be delayed on ExecutorStore, we maybe need to wait that the flow is updated
        flow = Await.until(() -> flowExecutorInterface.findByExecution(executor.getExecution()).orElse(null), Duration.ofMillis(100), Duration.ofMinutes(5));
    } catch (TimeoutException e) {
        return executor.withException(new Exception("Unable to find flow with namespace: '" + executor.getExecution().getNamespace() + "'" + ", id: '" + executor.getExecution().getFlowId() + "', " + "revision '" + executor.getExecution().getFlowRevision() + "'"), "joinFlow");
    }
    if (!withDefaults) {
        return executor.withFlow(flow);
    }
    try {
        flow = Template.injectTemplate(flow, executor.getExecution(), (namespace, id) -> this.templateExecutorInterface.findById(namespace, id).orElse(null));
    } catch (InternalException e) {
        log.warn("Failed to inject template", e);
    }
    Flow flowWithDefaults = taskDefaultService.injectDefaults(flow, executor.getExecution());
    return executor.withFlow(flowWithDefaults);
}