Search in sources :

Example 1 with DefaultKafkaConsumerConfiguration

use of io.micronaut.configuration.kafka.config.DefaultKafkaConsumerConfiguration in project micronaut-kafka by micronaut-projects.

the class KafkaConsumerProcessor method process.

@Override
public void process(BeanDefinition<?> beanDefinition, ExecutableMethod<?, ?> method) {
    List<AnnotationValue<Topic>> topicAnnotations = method.getDeclaredAnnotationValuesByType(Topic.class);
    final AnnotationValue<KafkaListener> consumerAnnotation = method.getAnnotation(KafkaListener.class);
    if (CollectionUtils.isEmpty(topicAnnotations)) {
        topicAnnotations = beanDefinition.getDeclaredAnnotationValuesByType(Topic.class);
    }
    if (consumerAnnotation == null || CollectionUtils.isEmpty(topicAnnotations)) {
        // No topics to consume
        return;
    }
    final Class<?> beanType = beanDefinition.getBeanType();
    String groupId = consumerAnnotation.stringValue("groupId").filter(StringUtils::isNotEmpty).orElseGet(() -> applicationConfiguration.getName().orElse(beanType.getName()));
    if (consumerAnnotation.isTrue("uniqueGroupId")) {
        groupId = groupId + "_" + UUID.randomUUID();
    }
    final String clientId = consumerAnnotation.stringValue("clientId").filter(StringUtils::isNotEmpty).orElseGet(() -> applicationConfiguration.getName().map(s -> s + '-' + NameUtils.hyphenate(beanType.getSimpleName())).orElse(null));
    final OffsetStrategy offsetStrategy = consumerAnnotation.enumValue("offsetStrategy", OffsetStrategy.class).orElse(OffsetStrategy.AUTO);
    final AbstractKafkaConsumerConfiguration<?, ?> consumerConfigurationDefaults = beanContext.findBean(AbstractKafkaConsumerConfiguration.class, Qualifiers.byName(groupId)).orElse(defaultConsumerConfiguration);
    final DefaultKafkaConsumerConfiguration<?, ?> consumerConfiguration = new DefaultKafkaConsumerConfiguration<>(consumerConfigurationDefaults);
    final Properties properties = createConsumerProperties(consumerAnnotation, consumerConfiguration, clientId, groupId, offsetStrategy);
    configureDeserializers(method, consumerConfiguration);
    submitConsumerThreads(method, clientId, groupId, offsetStrategy, topicAnnotations, consumerAnnotation, consumerConfiguration, properties, beanType);
}
Also used : AbstractKafkaConsumerConfiguration(io.micronaut.configuration.kafka.config.AbstractKafkaConsumerConfiguration) OffsetStrategy(io.micronaut.configuration.kafka.annotation.OffsetStrategy) Properties(java.util.Properties) KafkaListener(io.micronaut.configuration.kafka.annotation.KafkaListener) DefaultKafkaConsumerConfiguration(io.micronaut.configuration.kafka.config.DefaultKafkaConsumerConfiguration) AnnotationValue(io.micronaut.core.annotation.AnnotationValue) Topic(io.micronaut.configuration.kafka.annotation.Topic)

Example 2 with DefaultKafkaConsumerConfiguration

use of io.micronaut.configuration.kafka.config.DefaultKafkaConsumerConfiguration in project micronaut-kafka by micronaut-projects.

the class KafkaConsumerProcessor method configureDeserializers.

private void configureDeserializers(final ExecutableMethod<?, ?> method, final DefaultKafkaConsumerConfiguration consumerConfiguration) {
    final Properties properties = consumerConfiguration.getConfig();
    // figure out the Key deserializer
    final Argument<?> bodyArgument = findBodyArgument(method);
    if (!properties.containsKey(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG) && !consumerConfiguration.getKeyDeserializer().isPresent()) {
        final Optional<Argument<?>> keyArgument = Arrays.stream(method.getArguments()).filter(arg -> arg.isAnnotationPresent(KafkaKey.class)).findFirst();
        if (keyArgument.isPresent()) {
            consumerConfiguration.setKeyDeserializer(serdeRegistry.pickDeserializer(keyArgument.get()));
        } else {
            // noinspection SingleStatementInBlock
            if (bodyArgument != null && ConsumerRecord.class.isAssignableFrom(bodyArgument.getType())) {
                final Optional<Argument<?>> keyType = bodyArgument.getTypeVariable("K");
                if (keyType.isPresent()) {
                    consumerConfiguration.setKeyDeserializer(serdeRegistry.pickDeserializer(keyType.get()));
                } else {
                    consumerConfiguration.setKeyDeserializer(new ByteArrayDeserializer());
                }
            } else {
                consumerConfiguration.setKeyDeserializer(new ByteArrayDeserializer());
            }
        }
    }
    // figure out the Value deserializer
    if (!properties.containsKey(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG) && !consumerConfiguration.getValueDeserializer().isPresent()) {
        if (bodyArgument == null) {
            // noinspection SingleStatementInBlock
            consumerConfiguration.setValueDeserializer(new StringDeserializer());
        } else {
            if (ConsumerRecord.class.isAssignableFrom(bodyArgument.getType())) {
                final Optional<Argument<?>> valueType = bodyArgument.getTypeVariable("V");
                if (valueType.isPresent()) {
                    consumerConfiguration.setValueDeserializer(serdeRegistry.pickDeserializer(valueType.get()));
                } else {
                    consumerConfiguration.setValueDeserializer(new StringDeserializer());
                }
            } else {
                final boolean batch = method.isTrue(KafkaListener.class, "batch");
                consumerConfiguration.setValueDeserializer(serdeRegistry.pickDeserializer(batch ? getComponentType(bodyArgument) : bodyArgument));
            }
        }
    }
    debugDeserializationConfiguration(method, consumerConfiguration, properties);
}
Also used : Topic(io.micronaut.configuration.kafka.annotation.Topic) Publishers(io.micronaut.core.async.publisher.Publishers) Bindable(io.micronaut.core.bind.annotation.Bindable) Arrays(java.util.Arrays) KafkaAcknowledgement(io.micronaut.configuration.kafka.KafkaAcknowledgement) ConsumerRecords(org.apache.kafka.clients.consumer.ConsumerRecords) MessagingSystemException(io.micronaut.messaging.exceptions.MessagingSystemException) ConsumerRecordBinderRegistry(io.micronaut.configuration.kafka.bind.ConsumerRecordBinderRegistry) StringDeserializer(org.apache.kafka.common.serialization.StringDeserializer) Duration(java.time.Duration) Map(java.util.Map) OffsetCommitCallback(org.apache.kafka.clients.consumer.OffsetCommitCallback) ArgumentUtils(io.micronaut.core.util.ArgumentUtils) DefaultExecutableBinder(io.micronaut.core.bind.DefaultExecutableBinder) Singleton(jakarta.inject.Singleton) Set(java.util.Set) Acknowledgement(io.micronaut.messaging.Acknowledgement) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) RecordMetadata(org.apache.kafka.clients.producer.RecordMetadata) ErrorStrategy(io.micronaut.configuration.kafka.annotation.ErrorStrategy) ExecutableMethodProcessor(io.micronaut.context.processor.ExecutableMethodProcessor) StandardCharsets(java.nio.charset.StandardCharsets) KafkaKey(io.micronaut.configuration.kafka.annotation.KafkaKey) SerdeRegistry(io.micronaut.configuration.kafka.serde.SerdeRegistry) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) AnnotationValue(io.micronaut.core.annotation.AnnotationValue) AbstractKafkaConsumerConfiguration(io.micronaut.configuration.kafka.config.AbstractKafkaConsumerConfiguration) ProducerRegistry(io.micronaut.configuration.kafka.ProducerRegistry) MessageBody(io.micronaut.messaging.annotation.MessageBody) ConsumerRegistry(io.micronaut.configuration.kafka.ConsumerRegistry) RecordHeader(org.apache.kafka.common.header.internals.RecordHeader) KafkaMessage(io.micronaut.configuration.kafka.KafkaMessage) Nullable(io.micronaut.core.annotation.Nullable) ReturnType(io.micronaut.core.type.ReturnType) Argument(io.micronaut.core.type.Argument) Blocking(io.micronaut.core.annotation.Blocking) ByteArrayDeserializer(org.apache.kafka.common.serialization.ByteArrayDeserializer) Properties(java.util.Properties) Producer(org.apache.kafka.clients.producer.Producer) OffsetStrategy(io.micronaut.configuration.kafka.annotation.OffsetStrategy) Publisher(org.reactivestreams.Publisher) KafkaListenerException(io.micronaut.configuration.kafka.exceptions.KafkaListenerException) Mono(reactor.core.publisher.Mono) SendTo(io.micronaut.messaging.annotation.SendTo) Flux(reactor.core.publisher.Flux) CommitFailedException(org.apache.kafka.clients.consumer.CommitFailedException) TaskScheduler(io.micronaut.scheduling.TaskScheduler) BeanDefinition(io.micronaut.inject.BeanDefinition) BeanContext(io.micronaut.context.BeanContext) ArrayUtils(io.micronaut.core.util.ArrayUtils) LoggerFactory(org.slf4j.LoggerFactory) PreDestroy(javax.annotation.PreDestroy) TaskExecutors(io.micronaut.scheduling.TaskExecutors) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) KafkaListenerExceptionHandler(io.micronaut.configuration.kafka.exceptions.KafkaListenerExceptionHandler) ApplicationConfiguration(io.micronaut.runtime.ApplicationConfiguration) NameUtils(io.micronaut.core.naming.NameUtils) Consumer(org.apache.kafka.clients.consumer.Consumer) TopicPartition(org.apache.kafka.common.TopicPartition) WakeupException(org.apache.kafka.common.errors.WakeupException) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Qualifiers(io.micronaut.inject.qualifiers.Qualifiers) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) StringUtils(io.micronaut.core.util.StringUtils) ConsumerRebalanceListener(org.apache.kafka.clients.consumer.ConsumerRebalanceListener) List(java.util.List) ProducerFencedException(org.apache.kafka.common.errors.ProducerFencedException) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) Named(jakarta.inject.Named) KafkaListener(io.micronaut.configuration.kafka.annotation.KafkaListener) BatchConsumerRecordsBinderRegistry(io.micronaut.configuration.kafka.bind.batch.BatchConsumerRecordsBinderRegistry) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) ErrorStrategyValue(io.micronaut.configuration.kafka.annotation.ErrorStrategyValue) ScheduledExecutorTaskScheduler(io.micronaut.scheduling.ScheduledExecutorTaskScheduler) TransactionalProducerRegistry(io.micronaut.configuration.kafka.TransactionalProducerRegistry) KafkaDefaultConfiguration(io.micronaut.configuration.kafka.config.KafkaDefaultConfiguration) HashMap(java.util.HashMap) Scheduler(reactor.core.scheduler.Scheduler) Function(java.util.function.Function) ExecutableMethod(io.micronaut.inject.ExecutableMethod) HashSet(java.util.HashSet) ExecutableBinder(io.micronaut.core.bind.ExecutableBinder) Requires(io.micronaut.context.annotation.Requires) Schedulers(reactor.core.scheduler.Schedulers) BoundExecutable(io.micronaut.core.bind.BoundExecutable) ExecutorService(java.util.concurrent.ExecutorService) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) OffsetReset(io.micronaut.configuration.kafka.annotation.OffsetReset) NonNull(io.micronaut.core.annotation.NonNull) IsolationLevel(org.apache.kafka.common.IsolationLevel) CollectionUtils(io.micronaut.core.util.CollectionUtils) DefaultKafkaConsumerConfiguration(io.micronaut.configuration.kafka.config.DefaultKafkaConsumerConfiguration) Collections(java.util.Collections) ConsumerAware(io.micronaut.configuration.kafka.ConsumerAware) Argument(io.micronaut.core.type.Argument) StringDeserializer(org.apache.kafka.common.serialization.StringDeserializer) Properties(java.util.Properties) ByteArrayDeserializer(org.apache.kafka.common.serialization.ByteArrayDeserializer) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord)

Aggregations

KafkaListener (io.micronaut.configuration.kafka.annotation.KafkaListener)2 OffsetStrategy (io.micronaut.configuration.kafka.annotation.OffsetStrategy)2 Topic (io.micronaut.configuration.kafka.annotation.Topic)2 AbstractKafkaConsumerConfiguration (io.micronaut.configuration.kafka.config.AbstractKafkaConsumerConfiguration)2 DefaultKafkaConsumerConfiguration (io.micronaut.configuration.kafka.config.DefaultKafkaConsumerConfiguration)2 AnnotationValue (io.micronaut.core.annotation.AnnotationValue)2 ConsumerAware (io.micronaut.configuration.kafka.ConsumerAware)1 ConsumerRegistry (io.micronaut.configuration.kafka.ConsumerRegistry)1 KafkaAcknowledgement (io.micronaut.configuration.kafka.KafkaAcknowledgement)1 KafkaMessage (io.micronaut.configuration.kafka.KafkaMessage)1 ProducerRegistry (io.micronaut.configuration.kafka.ProducerRegistry)1 TransactionalProducerRegistry (io.micronaut.configuration.kafka.TransactionalProducerRegistry)1 ErrorStrategy (io.micronaut.configuration.kafka.annotation.ErrorStrategy)1 ErrorStrategyValue (io.micronaut.configuration.kafka.annotation.ErrorStrategyValue)1 KafkaKey (io.micronaut.configuration.kafka.annotation.KafkaKey)1 OffsetReset (io.micronaut.configuration.kafka.annotation.OffsetReset)1 ConsumerRecordBinderRegistry (io.micronaut.configuration.kafka.bind.ConsumerRecordBinderRegistry)1 BatchConsumerRecordsBinderRegistry (io.micronaut.configuration.kafka.bind.batch.BatchConsumerRecordsBinderRegistry)1 KafkaDefaultConfiguration (io.micronaut.configuration.kafka.config.KafkaDefaultConfiguration)1 KafkaListenerException (io.micronaut.configuration.kafka.exceptions.KafkaListenerException)1