Search in sources :

Example 11 with Trigger

use of io.kestra.core.models.triggers.Trigger in project kestra by kestra-io.

the class ElasticSearchTriggerRepositoryTest method all.

@Test
void all() {
    Trigger.TriggerBuilder<?, ?> builder = trigger();
    Optional<Trigger> find = triggerRepository.findLast(builder.build());
    assertThat(find.isPresent(), is(false));
    Trigger save = triggerRepository.save(builder.build());
    find = triggerRepository.findLast(save);
    assertThat(find.isPresent(), is(true));
    assertThat(find.get().getExecutionId(), is(save.getExecutionId()));
    save = triggerRepository.save(builder.executionId(IdUtils.create()).build());
    find = triggerRepository.findLast(save);
    assertThat(find.isPresent(), is(true));
    assertThat(find.get().getExecutionId(), is(save.getExecutionId()));
    triggerRepository.save(trigger().build());
    triggerRepository.save(trigger().build());
    triggerRepository.save(trigger().build());
    List<Trigger> all = triggerRepository.findAll();
    assertThat(all.size(), is(4));
}
Also used : Trigger(io.kestra.core.models.triggers.Trigger) Test(org.junit.jupiter.api.Test) MicronautTest(io.micronaut.test.extensions.junit5.annotation.MicronautTest)

Example 12 with Trigger

use of io.kestra.core.models.triggers.Trigger in project kestra by kestra-io.

the class ExecutorFlowLast method topology.

public StreamsBuilder topology() {
    StreamsBuilder builder = new KafkaStreamsBuilder();
    // last global KTable
    GlobalKTable<String, Flow> flowGlobalKTable = builder.globalTable(kafkaAdminService.getTopicName(KafkaStreamSourceService.TOPIC_FLOWLAST), Consumed.with(Serdes.String(), JsonSerde.of(Flow.class)).withName("GlobalKTable.FlowLast"), Materialized.<String, Flow, KeyValueStore<Bytes, byte[]>>as("last").withKeySerde(Serdes.String()).withValueSerde(JsonSerde.of(Flow.class)));
    // stream
    KStream<String, Flow> stream = builder.stream(kafkaAdminService.getTopicName(Flow.class), Consumed.with(Serdes.String(), JsonSerde.of(Flow.class, false)).withName("Stream.Flow"));
    // logs
    stream = KafkaStreamSourceService.logIfEnabled(log, stream, (key, value) -> log.trace("Flow in '{}.{}' with revision {}", value.getNamespace(), value.getId(), value.getRevision()), "Main");
    // join with previous if more recent revision
    KStream<String, ExecutorFlowLast.FlowWithPrevious> streamWithPrevious = stream.filter((key, value) -> value != null, Named.as("Main.notNull")).selectKey((key, value) -> value.uidWithoutRevision(), Named.as("Main.selectKey")).leftJoin(flowGlobalKTable, (key, value) -> key, (readOnlyKey, current, previous) -> {
        if (previous == null) {
            return new ExecutorFlowLast.FlowWithPrevious(current, null);
        } else if (current.getRevision() < previous.getRevision()) {
            return null;
        } else {
            return new ExecutorFlowLast.FlowWithPrevious(current, previous);
        }
    }, Named.as("Main.join")).filter((key, value) -> value != null, Named.as("Main.joinNotNull"));
    // remove triggers
    streamWithPrevious.flatMap((key, value) -> {
        List<AbstractTrigger> deletedTriggers = new ArrayList<>();
        if (value.getFlow().isDeleted()) {
            deletedTriggers = ListUtils.emptyOnNull(value.getFlow().getTriggers());
        } else if (value.getPrevious() != null) {
            deletedTriggers = FlowService.findRemovedTrigger(value.getFlow(), value.getPrevious());
        }
        return deletedTriggers.stream().map(t -> new KeyValue<>(queueService.key(Trigger.of(value.getFlow(), t)), (Trigger) null)).collect(Collectors.toList());
    }, Named.as("DeleteTrigger.flatMap")).to(kafkaAdminService.getTopicName(Trigger.class), Produced.with(Serdes.String(), JsonSerde.of(Trigger.class)).withName("To.Trigger"));
    // send to last and don't drop deleted flow in order to keep last version
    streamWithPrevious.map((key, value) -> new KeyValue<>(value.getFlow().uidWithoutRevision(), value.getFlow()), Named.as("Main.Map")).to(kafkaAdminService.getTopicName(KafkaStreamSourceService.TOPIC_FLOWLAST), Produced.with(Serdes.String(), JsonSerde.of(Flow.class)).withName("To.FlowLast"));
    return builder;
}
Also used : Getter(lombok.Getter) ListUtils(io.kestra.core.utils.ListUtils) KafkaAdminService(io.kestra.runner.kafka.services.KafkaAdminService) AbstractTrigger(io.kestra.core.models.triggers.AbstractTrigger) QueueService(io.kestra.core.queues.QueueService) ArrayList(java.util.ArrayList) Jacksonized(lombok.extern.jackson.Jacksonized) FlowService(io.kestra.core.services.FlowService) KeyValueStore(org.apache.kafka.streams.state.KeyValueStore) Serdes(org.apache.kafka.common.serialization.Serdes) KafkaStreamSourceService(io.kestra.runner.kafka.services.KafkaStreamSourceService) KafkaStreamsBuilder(io.kestra.runner.kafka.services.KafkaStreamsBuilder) KafkaQueueEnabled(io.kestra.runner.kafka.KafkaQueueEnabled) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) org.apache.kafka.streams.kstream(org.apache.kafka.streams.kstream) KeyValue(org.apache.kafka.streams.KeyValue) Singleton(jakarta.inject.Singleton) Trigger(io.kestra.core.models.triggers.Trigger) JsonSerde(io.kestra.runner.kafka.serializers.JsonSerde) Collectors(java.util.stream.Collectors) Bytes(org.apache.kafka.common.utils.Bytes) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Flow(io.kestra.core.models.flows.Flow) Inject(jakarta.inject.Inject) AllArgsConstructor(lombok.AllArgsConstructor) KeyValue(org.apache.kafka.streams.KeyValue) Flow(io.kestra.core.models.flows.Flow) KafkaStreamsBuilder(io.kestra.runner.kafka.services.KafkaStreamsBuilder) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) Bytes(org.apache.kafka.common.utils.Bytes) AbstractTrigger(io.kestra.core.models.triggers.AbstractTrigger) Trigger(io.kestra.core.models.triggers.Trigger) ArrayList(java.util.ArrayList) List(java.util.List) KafkaStreamsBuilder(io.kestra.runner.kafka.services.KafkaStreamsBuilder)

Aggregations

Trigger (io.kestra.core.models.triggers.Trigger)12 AbstractTrigger (io.kestra.core.models.triggers.AbstractTrigger)6 Execution (io.kestra.core.models.executions.Execution)4 Inject (jakarta.inject.Inject)4 Singleton (jakarta.inject.Singleton)4 Slf4j (lombok.extern.slf4j.Slf4j)4 Flow (io.kestra.core.models.flows.Flow)3 ConditionService (io.kestra.core.services.ConditionService)3 ZonedDateTime (java.time.ZonedDateTime)3 Collectors (java.util.stream.Collectors)3 AllArgsConstructor (lombok.AllArgsConstructor)3 Getter (lombok.Getter)3 Throwables (com.google.common.base.Throwables)2 com.google.common.util.concurrent (com.google.common.util.concurrent)2 InternalException (io.kestra.core.exceptions.InternalException)2 MetricRegistry (io.kestra.core.metrics.MetricRegistry)2 ConditionContext (io.kestra.core.models.conditions.ConditionContext)2 PollingTriggerInterface (io.kestra.core.models.triggers.PollingTriggerInterface)2 TriggerContext (io.kestra.core.models.triggers.TriggerContext)2 QueueFactoryInterface (io.kestra.core.queues.QueueFactoryInterface)2