Examples with TbProtoQueueMsg org.thingsboard.server.queue.common.TbProtoQueueMsg used on opensource projects

Example 11 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class DefaultTbClusterService method broadcast.

private void broadcast(ToTransportMsg transportMsg, TbQueueCallback callback) {
    TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> toTransportNfProducer = producerProvider.getTransportNotificationsMsgProducer();
    Set<String> tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT);
    TbQueueCallback proxyCallback = callback != null ? new MultipleTbQueueCallbackWrapper(tbTransportServices.size(), callback) : null;
    for (String transportServiceId : tbTransportServices) {
        TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId);
        toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), proxyCallback);
        toTransportNfs.incrementAndGet();
    }
}

Example 12 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class DefaultTbClusterService method broadcast.

private void broadcast(ComponentLifecycleMsg msg) {
    byte[] msgBytes = encodingService.encode(msg);
    TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer();
    Set<String> tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE));
    EntityType entityType = msg.getEntityId().getEntityType();
    if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) || entityType.equals(EntityType.DEVICE_PROFILE) || entityType.equals(EntityType.API_USAGE_STATE) || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.EDGE)) {
        TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer();
        Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE);
        for (String serviceId : tbCoreServices) {
            TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
            ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build();
            toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null);
            toCoreNfs.incrementAndGet();
        }
        // No need to push notifications twice
        tbRuleEngineServices.removeAll(tbCoreServices);
    }
    for (String serviceId : tbRuleEngineServices) {
        TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId);
        ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build();
        toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null);
        toRuleEngineNfs.incrementAndGet();
    }
}

Example 13 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class DefaultTbCoreConsumerService method launchMainConsumers.

@Override
protected void launchMainConsumers() {
    consumersExecutor.submit(() -> {
        while (!stopped) {
            try {
                List<TbProtoQueueMsg<ToCoreMsg>> msgs = mainConsumer.poll(pollDuration);
                if (msgs.isEmpty()) {
                    continue;
                }
                List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).collect(Collectors.toList());
                ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
                CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
                TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>(processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
                PendingMsgHolder pendingMsgHolder = new PendingMsgHolder();
                Future<?> packSubmitFuture = consumersExecutor.submit(() -> {
                    orderedMsgList.forEach((element) -> {
                        UUID id = element.getUuid();
                        TbProtoQueueMsg<ToCoreMsg> msg = element.getMsg();
                        log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
                        TbCallback callback = new TbPackCallback<>(id, ctx);
                        try {
                            ToCoreMsg toCoreMsg = msg.getValue();
                            pendingMsgHolder.setToCoreMsg(toCoreMsg);
                            if (toCoreMsg.hasToSubscriptionMgrMsg()) {
                                log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg());
                                forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback);
                            } else if (toCoreMsg.hasToDeviceActorMsg()) {
                                log.trace("[{}] Forwarding message to device actor {}", id, toCoreMsg.getToDeviceActorMsg());
                                forwardToDeviceActor(toCoreMsg.getToDeviceActorMsg(), callback);
                            } else if (toCoreMsg.hasDeviceStateServiceMsg()) {
                                log.trace("[{}] Forwarding message to state service {}", id, toCoreMsg.getDeviceStateServiceMsg());
                                forwardToStateService(toCoreMsg.getDeviceStateServiceMsg(), callback);
                            } else if (toCoreMsg.hasEdgeNotificationMsg()) {
                                log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg());
                                forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback);
                            } else if (!toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) {
                                Optional<TbActorMsg> actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray());
                                if (actorMsg.isPresent()) {
                                    TbActorMsg tbActorMsg = actorMsg.get();
                                    if (tbActorMsg.getMsgType().equals(MsgType.DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG)) {
                                        tbCoreDeviceRpcService.forwardRpcRequestToDeviceActor((ToDeviceRpcRequestActorMsg) tbActorMsg);
                                    } else {
                                        log.trace("[{}] Forwarding message to App Actor {}", id, actorMsg.get());
                                        actorContext.tell(actorMsg.get());
                                    }
                                }
                                callback.onSuccess();
                            }
                        } catch (Throwable e) {
                            log.warn("[{}] Failed to process message: {}", id, msg, e);
                            callback.onFailure(e);
                        }
                    });
                });
                if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) {
                    if (!packSubmitFuture.isDone()) {
                        packSubmitFuture.cancel(true);
                        ToCoreMsg lastSubmitMsg = pendingMsgHolder.getToCoreMsg();
                        log.info("Timeout to process message: {}", lastSubmitMsg);
                    }
                    ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue()));
                    ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue()));
                }
                mainConsumer.commit();
            } catch (Exception e) {
                if (!stopped) {
                    log.warn("Failed to obtain messages from queue.", e);
                    try {
                        Thread.sleep(pollDuration);
                    } catch (InterruptedException e2) {
                        log.trace("Failed to wait until the server has capacity to handle new requests", e2);
                    }
                }
            }
        }
        log.info("TB Core Consumer stopped.");
    });
}

Example 14 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class AbstractConsumerService method launchNotificationsConsumer.

protected void launchNotificationsConsumer() {
    notificationsConsumerExecutor.submit(() -> {
        while (!stopped) {
            try {
                List<TbProtoQueueMsg<N>> msgs = nfConsumer.poll(getNotificationPollDuration());
                if (msgs.isEmpty()) {
                    continue;
                }
                ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = msgs.stream().collect(Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity()));
                CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
                TbPackProcessingContext<TbProtoQueueMsg<N>> ctx = new TbPackProcessingContext<>(processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
                pendingMap.forEach((id, msg) -> {
                    log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue());
                    TbCallback callback = new TbPackCallback<>(id, ctx);
                    try {
                        handleNotification(id, msg, callback);
                    } catch (Throwable e) {
                        log.warn("[{}] Failed to process notification: {}", id, msg, e);
                        callback.onFailure(e);
                    }
                });
                if (!processingTimeoutLatch.await(getNotificationPackProcessingTimeout(), TimeUnit.MILLISECONDS)) {
                    ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process notification: {}", id, msg.getValue()));
                    ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process notification: {}", id, msg.getValue()));
                }
                nfConsumer.commit();
            } catch (Exception e) {
                if (!stopped) {
                    log.warn("Failed to obtain notifications from queue.", e);
                    try {
                        Thread.sleep(getNotificationPollDuration());
                    } catch (InterruptedException e2) {
                        log.trace("Failed to wait until the server has capacity to handle new notifications", e2);
                    }
                }
            }
        }
        log.info("TB Notifications Consumer stopped.");
    });
}

Example 15 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class InMemoryTbTransportQueueFactory method createTransportApiRequestTemplate.

@Override
public TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> createTransportApiRequestTemplate() {
    InMemoryTbQueueProducer<TbProtoQueueMsg<TransportApiRequestMsg>> producerTemplate = new InMemoryTbQueueProducer<>(transportApiSettings.getRequestsTopic());
    InMemoryTbQueueConsumer<TbProtoQueueMsg<TransportApiResponseMsg>> consumerTemplate = new InMemoryTbQueueConsumer<>(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId());
    DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> templateBuilder = DefaultTbQueueRequestTemplate.builder();
    templateBuilder.queueAdmin(new TbQueueAdmin() {

        @Override
        public void createTopicIfNotExists(String topic) {
        }

        @Override
        public void destroy() {
        }
    });
    templateBuilder.requestTemplate(producerTemplate);
    templateBuilder.responseTemplate(consumerTemplate);
    templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests());
    templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout());
    templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval());
    return templateBuilder.build();
}