Examples with Trigger io.kestra.core.models.triggers.Trigger used on opensource projects

Example 6 with Trigger

use of io.kestra.core.models.triggers.Trigger in project kestra by kestra-io.

the class AbstractScheduler method handle.

private void handle() {
    if (!this.isReady) {
        log.warn("Scheduler is not ready, waiting");
    }
    metricRegistry.counter(MetricRegistry.SCHEDULER_LOOP_COUNT).increment();
    ZonedDateTime now = now();
    synchronized (this) {
        if (log.isDebugEnabled()) {
            log.debug("Scheduler next iteration for {} with {} schedulables of {} flows", now, schedulable.size(), this.flowListeners.flows().size());
        }
        // get all that is ready from evaluation
        List<FlowWithPollingTriggerNextDate> readyForEvaluate = schedulable.stream().filter(f -> conditionService.isValid(f.getFlow(), f.getTrigger(), f.getConditionContext())).map(flowWithTrigger -> FlowWithPollingTrigger.builder().flow(flowWithTrigger.getFlow()).trigger(flowWithTrigger.getTrigger()).pollingTrigger((PollingTriggerInterface) flowWithTrigger.getTrigger()).conditionContext(flowWithTrigger.getConditionContext()).triggerContext(TriggerContext.builder().namespace(flowWithTrigger.getFlow().getNamespace()).flowId(flowWithTrigger.getFlow().getId()).flowRevision(flowWithTrigger.getFlow().getRevision()).triggerId(flowWithTrigger.getTrigger().getId()).date(now()).build()).build()).filter(f -> this.isEvaluationInterval(f, now)).filter(f -> this.isExecutionNotRunning(f, now)).map(f -> {
            synchronized (this) {
                Trigger lastTrigger = this.getLastTrigger(f, now);
                return FlowWithPollingTriggerNextDate.of(f, f.getPollingTrigger().nextEvaluationDate(f.getConditionContext(), Optional.of(lastTrigger)));
            }
        }).filter(Objects::nonNull).collect(Collectors.toList());
        if (log.isDebugEnabled()) {
            log.debug("Scheduler will evaluate for {} with {} readyForEvaluate of {} schedulables", now, readyForEvaluate.size(), schedulable.size());
        }
        metricRegistry.counter(MetricRegistry.SCHEDULER_EVALUATE_COUNT).increment(readyForEvaluate.size());
        // submit ready one to cached thread pool
        readyForEvaluate.forEach(f -> {
            schedulableNextDate.put(f.getTriggerContext().uid(), f);
            if (f.getPollingTrigger().getInterval() == null) {
                try {
                    this.handleEvaluatePollingTriggerResult(this.evaluatePollingTrigger(f));
                } catch (Exception e) {
                    AbstractScheduler.logError(f, e);
                }
            } else {
                this.addToRunning(f.getTriggerContext(), now);
                ListenableFuture<SchedulerExecutionWithTrigger> result = cachedExecutor.submit(() -> this.evaluatePollingTrigger(f));
                Futures.addCallback(result, new EvaluateFuture(this, f), cachedExecutor);
            }
        });
    }
}

Example 7 with Trigger

use of io.kestra.core.models.triggers.Trigger in project kestra by kestra-io.

the class AbstractScheduler method isExecutionNotRunning.

private boolean isExecutionNotRunning(FlowWithPollingTrigger f, ZonedDateTime now) {
    Trigger lastTrigger = this.getLastTrigger(f, now);
    if (lastTrigger.getExecutionId() == null) {
        return true;
    }
    Optional<Execution> execution = executionState.findById(lastTrigger.getExecutionId());
    // executionState hasn't received the execution, we skip
    if (execution.isEmpty()) {
        if (lastTrigger.getUpdatedDate() != null) {
            metricRegistry.timer(MetricRegistry.SCHEDULER_EXECUTION_MISSING_DURATION, metricRegistry.tags(lastTrigger)).record(Duration.between(lastTrigger.getUpdatedDate(), Instant.now()));
        }
        if (lastTrigger.getUpdatedDate() == null || lastTrigger.getUpdatedDate().plusSeconds(60).isBefore(Instant.now())) {
            log.warn("Execution '{}' for flow '{}.{}' is not found, schedule is blocked since {}", lastTrigger.getExecutionId(), lastTrigger.getNamespace(), lastTrigger.getFlowId(), lastTrigger.getUpdatedDate());
        }
        return false;
    }
    // the scheduler will clean the execution from the trigger and we don't keep only terminated state as an end.
    if (log.isDebugEnabled()) {
        if (lastTrigger.getUpdatedDate() != null) {
            metricRegistry.timer(MetricRegistry.SCHEDULER_EXECUTION_RUNNING_DURATION, metricRegistry.tags(lastTrigger)).record(Duration.between(lastTrigger.getUpdatedDate(), Instant.now()));
        }
        log.debug("Execution '{}' for flow '{}.{}' is still '{}', waiting for next backfill", lastTrigger.getExecutionId(), lastTrigger.getNamespace(), lastTrigger.getFlowId(), execution.get().getState().getCurrent());
    }
    return false;
}

Example 8 with Trigger

use of io.kestra.core.models.triggers.Trigger in project kestra by kestra-io.

the class AbstractScheduler method computeSchedulable.

private void computeSchedulable(List<Flow> flows) {
    schedulableNextDate = new HashMap<>();
    this.schedulable = flows.stream().filter(flow -> flow.getTriggers() != null && flow.getTriggers().size() > 0).flatMap(flow -> flow.getTriggers().stream().map(trigger -> {
        RunContext runContext = runContextFactory.of(flow, trigger);
        return new FlowWithTrigger(flow, trigger, runContext, conditionService.conditionContext(runContext, flow, null));
    })).filter(flowWithTrigger -> flowWithTrigger.getTrigger() instanceof PollingTriggerInterface).collect(Collectors.toList());
}

Example 9 with Trigger

use of io.kestra.core.models.triggers.Trigger in project kestra by kestra-io.

the class DefaultScheduler method run.

@SuppressWarnings("unchecked")
@Override
public void run() {
    flowListeners.run();
    QueueInterface<Execution> executionQueue = applicationContext.getBean(QueueInterface.class, Qualifiers.byName(QueueFactoryInterface.EXECUTION_NAMED));
    QueueInterface<Trigger> triggerQueue = applicationContext.getBean(QueueInterface.class, Qualifiers.byName(QueueFactoryInterface.TRIGGER_NAMED));
    executionQueue.receive(execution -> {
        if (execution.getState().getCurrent().isTerninated() && this.watchingTrigger.containsKey(execution.getId())) {
            Trigger trigger = watchingTrigger.get(execution.getId());
            triggerQueue.emit(trigger.resetExecution());
            triggerState.save(trigger.resetExecution());
        }
    });
    triggerQueue.receive(trigger -> {
        if (trigger.getExecutionId() != null) {
            this.watchingTrigger.put(trigger.getExecutionId(), trigger);
        }
    });
    super.run();
}

Example 10 with Trigger

use of io.kestra.core.models.triggers.Trigger in project kestra by kestra-io.

the class KafkaScheduler method saveLastTriggerAndEmitExecution.

/**
 * We saved the trigger in a local hash map that will be clean by {@link GlobalStateProcessor}.
 * The scheduler trust the STATESTORE_TRIGGER to know if a running execution exists. Since the store is filled async,
 * this can lead to empty trigger and launch of concurrent job.
 *
 * @param executionWithTrigger the execution trigger to save
 */
protected synchronized void saveLastTriggerAndEmitExecution(SchedulerExecutionWithTrigger executionWithTrigger) {
    Trigger trigger = Trigger.of(executionWithTrigger.getTriggerContext(), executionWithTrigger.getExecution());
    kafkaProducer.beginTransaction();
    this.kafkaProducer.send(new ProducerRecord<>(topicsConfigTrigger.getName(), this.queueService.key(trigger), trigger));
    this.kafkaProducer.send(new ProducerRecord<>(topicsConfigExecution.getName(), this.queueService.key(executionWithTrigger.getExecution()), executionWithTrigger.getExecution()));
    this.triggerLock.put(trigger.uid(), trigger);
    kafkaProducer.commitTransaction();
}