Examples with MqDeserializer com.hummer.message.common.deserializer.MqDeserializer used on opensource projects

Example 1 with MqDeserializer

use of com.hummer.message.common.deserializer.MqDeserializer in project hummer-framework by hummer-team.

the class KafkaConsumerTaskHolder method readMessage.

private Tuple.TupleTwo<List<MessageData<E>>, List<RecordOffsetMetadata>> readMessage(final ConsumerRecords<String, Bytes> records) {
    int size = records.count();
    List<MessageData<E>> messageList = Lists.newArrayListWithCapacity(size);
    List<RecordOffsetMetadata> offsetList = Lists.newArrayListWithCapacity(size);
    MqDeserializer deserializer = DeserializerFactory.deserializer(ConsumerProperties.getDeserializerType());
    Iterator<ConsumerRecord<String, Bytes>> recordIterator = records.iterator();
    while (recordIterator.hasNext()) {
        ConsumerRecord<String, Bytes> recordItem = recordIterator.next();
        if (recordItem != null) {
            // message append list
            try {
                E e = deserializer.deserialize(recordItem.value().get(), this.metadata.getMessageHandler().messageType(), recordItem.topic());
                MessageData<E> messageData = new MessageData<>();
                messageData.setData(e);
                messageData.setQueueOffset(recordItem.offset());
                messageData.setQueueId(recordItem.partition());
                messageData.setProperties(headerToMap(recordItem.headers()));
                messageList.add(messageData);
            } catch (Exception e) {
                LOGGER.warn("topic {} message deserialize {} error ", metadata.getTopicId(), deserializer, e);
            }
            // offset append list if enable batch commit
            if (metadata.getCommitBatchSize() > 0) {
                offsetList.add(RecordOffsetMetadata.builder().offset(recordItem.offset()).topicId(recordItem.topic()).partition(recordItem.partition()).build());
            }
        }
    }
    return new Tuple.TupleTwo<>(messageList, offsetList);
}

Example 2 with MqDeserializer

use of com.hummer.message.common.deserializer.MqDeserializer in project hummer-framework by hummer-team.

the class RabbitMqConsumer method start.

public void start() throws IOException, NoSuchAlgorithmException, KeyManagementException, URISyntaxException, TimeoutException {
    createConnection();
    channel = connection.createChannel();
    channel.queueDeclare(queueConfig.getQueueName(), queueConfig.isDurable(), queueConfig.isExclusive(), queueConfig.isAutoDelete(), queueConfig.getArguments());
    if (StringUtils.isNotEmpty(queueConfig.getRouteKey())) {
        channel.queueBind(queueConfig.getQueueName(), queueConfig.getExchange(), queueConfig.getRouteKey());
    }
    channel.addShutdownListener(new ShutdownListener() {

        @Override
        public void shutdownCompleted(ShutdownSignalException e) {
            if (!exitSignal) {
                log.warn("mq shutdown completed, reason is ", e);
            }
        }
    });
    channel.addReturnListener(new ReturnListener() {

        @Override
        public void handleReturn(int replyCode, String replyText, String exchange, String routingKey, AMQP.BasicProperties properties, byte[] body) throws IOException {
            log.info("consumer return {}", replyText);
        }
    });
    channel.basicQos(queueConfig.getQos());
    MqDeserializer mqDeserializer = DeserializerFactory.deserializer(queueConfig.getSerializerType());
    consumerTag = channel.basicConsume(queueConfig.getQueueName(), queueConfig.isAutoAck(), new DeliverCallback() {

        @Override
        public void handle(String consumerTag, Delivery message) throws IOException {
            T data;
            try {
                data = mqDeserializer.deserialize(message.getBody(), handler.messageType());
            } catch (Exception e) {
                log.error("deserialize message failed, topic is {} message type is {}", topic, handler.messageType(), e);
                return;
            }
            MessageData<T> messageData = new MessageData<>();
            messageData.setData(data);
            messageData.setQueueOffset(message.getEnvelope().getDeliveryTag());
            messageData.setProperties(Optional.ofNullable(message.getProperties().getHeaders()).orElse(Collections.emptyMap()).entrySet().stream().collect(Collectors.toConcurrentMap(Map.Entry::getKey, e -> String.valueOf(e.getValue()))));
            ListenableFuture<Long> future = ((ListeningExecutorService) executorService).submit(() -> handler.handlerAndReturnOffset(messageData));
            Futures.addCallback(future, new FutureCallback<>() {

                @Override
                public void onSuccess(@Nullable Long offset) {
                    ack(offset);
                }

                @Override
                public void onFailure(Throwable throwable) {
                    log.warn("business handler message failed {}", topic, throwable);
                }
            }, executorService);
        }
    }, new CancelCallback() {

        @Override
        public void handle(String consumerTag) throws IOException {
            log.warn("consumer cancel {}", consumerTag);
        }
    }, new ConsumerShutdownSignalCallback() {

        @Override
        public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) {
            log.warn("consumer shutdown {} ", consumerTag, sig);
        }
    });
    log.info("consumer {} start ok,queueConfig is {}", consumerTag, queueConfig);
}