Examples with TbProtoQueueMsg org.thingsboard.server.queue.common.TbProtoQueueMsg used on opensource projects

Example 6 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class DefaultTbRuleEngineConsumerService method repartitionTopicWithConsumerPerPartition.

void repartitionTopicWithConsumerPerPartition(final String queueName) {
    if (stopped) {
        return;
    }
    TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueName);
    Queue<Set<TopicPartitionInfo>> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue();
    if (subscribeQueue.isEmpty()) {
        return;
    }
    if (tbTopicWithConsumerPerPartition.getLock().tryLock()) {
        try {
            Set<TopicPartitionInfo> partitions = null;
            while (!subscribeQueue.isEmpty()) {
                partitions = subscribeQueue.poll();
            }
            if (partitions == null) {
                return;
            }
            Set<TopicPartitionInfo> addedPartitions = new HashSet<>(partitions);
            ConcurrentMap<TopicPartitionInfo, TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> consumers = tbTopicWithConsumerPerPartition.getConsumers();
            addedPartitions.removeAll(consumers.keySet());
            log.info("calculated addedPartitions {}", addedPartitions);
            Set<TopicPartitionInfo> removedPartitions = new HashSet<>(consumers.keySet());
            removedPartitions.removeAll(partitions);
            log.info("calculated removedPartitions {}", removedPartitions);
            removedPartitions.forEach((tpi) -> {
                removeConsumerForTopicByTpi(queueName, consumers, tpi);
            });
            addedPartitions.forEach((tpi) -> {
                log.info("[{}] Adding consumer for topic: {}", queueName, tpi);
                TbRuleEngineQueueConfiguration configuration = consumerConfigurations.get(queueName);
                TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration);
                consumers.put(tpi, consumer);
                launchConsumer(consumer, consumerConfigurations.get(queueName), consumerStats.get(queueName), "" + queueName + "-" + tpi.getPartition().orElse(-999999));
                consumer.subscribe(Collections.singleton(tpi));
            });
        } finally {
            tbTopicWithConsumerPerPartition.getLock().unlock();
        }
    } else {
        // reschedule later
        scheduleTopicRepartition(queueName);
    }
}

Example 7 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class DefaultTbRuleEngineConsumerService method consumerLoop.

void consumerLoop(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats, String threadSuffix) {
    updateCurrentThreadName(threadSuffix);
    while (!stopped && !consumer.isStopped()) {
        try {
            List<TbProtoQueueMsg<ToRuleEngineMsg>> msgs = consumer.poll(pollDuration);
            if (msgs.isEmpty()) {
                continue;
            }
            final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(configuration);
            final TbRuleEngineProcessingStrategy ackStrategy = getAckStrategy(configuration);
            submitStrategy.init(msgs);
            while (!stopped) {
                TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(configuration.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs());
                submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> submitMessage(configuration, stats, ctx, id, msg)));
                final boolean timeout = !ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS);
                TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, ctx);
                if (timeout) {
                    printFirstOrAll(configuration, ctx, ctx.getPendingMap(), "Timeout");
                }
                if (!ctx.getFailedMap().isEmpty()) {
                    printFirstOrAll(configuration, ctx, ctx.getFailedMap(), "Failed");
                }
                ctx.printProfilerStats();
                TbRuleEngineProcessingDecision decision = ackStrategy.analyze(result);
                if (statsEnabled) {
                    stats.log(result, decision.isCommit());
                }
                ctx.cleanup();
                if (decision.isCommit()) {
                    submitStrategy.stop();
                    break;
                } else {
                    submitStrategy.update(decision.getReprocessMap());
                }
            }
            consumer.commit();
        } catch (Exception e) {
            if (!stopped) {
                log.warn("Failed to process messages from queue.", e);
                try {
                    Thread.sleep(pollDuration);
                } catch (InterruptedException e2) {
                    log.trace("Failed to wait until the server has capacity to handle new requests", e2);
                }
            }
        }
    }
    log.info("TB Rule Engine Consumer stopped.");
}

Example 8 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class DefaultTransportService method process.

@Override
public void process(TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg requestMsg, TransportServiceCallback<GetOrCreateDeviceFromGatewayResponse> callback) {
    TbProtoQueueMsg<TransportApiRequestMsg> protoMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), TransportApiRequestMsg.newBuilder().setGetOrCreateDeviceRequestMsg(requestMsg).build());
    log.trace("Processing msg: {}", requestMsg);
    ListenableFuture<GetOrCreateDeviceFromGatewayResponse> response = Futures.transform(transportApiRequestTemplate.send(protoMsg), tmp -> {
        TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg msg = tmp.getValue().getGetOrCreateDeviceResponseMsg();
        GetOrCreateDeviceFromGatewayResponse.GetOrCreateDeviceFromGatewayResponseBuilder result = GetOrCreateDeviceFromGatewayResponse.builder();
        if (msg.hasDeviceInfo()) {
            TransportDeviceInfo tdi = getTransportDeviceInfo(msg.getDeviceInfo());
            result.deviceInfo(tdi);
            ByteString profileBody = msg.getProfileBody();
            if (profileBody != null && !profileBody.isEmpty()) {
                result.deviceProfile(deviceProfileCache.getOrCreate(tdi.getDeviceProfileId(), profileBody));
            }
        }
        return result.build();
    }, MoreExecutors.directExecutor());
    AsyncCallbackTemplate.withCallback(response, callback::onSuccess, callback::onError, transportCallbackExecutor);
}

Example 9 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class DefaultSubscriptionManagerService method toProto.

private TbProtoQueueMsg<ToCoreNotificationMsg> toProto(TbSubscription subscription, Alarm alarm, boolean deleted) {
    TbAlarmSubscriptionUpdateProto.Builder builder = TbAlarmSubscriptionUpdateProto.newBuilder();
    builder.setSessionId(subscription.getSessionId());
    builder.setSubscriptionId(subscription.getSubscriptionId());
    builder.setAlarm(JacksonUtil.toString(alarm));
    builder.setDeleted(deleted);
    ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg(LocalSubscriptionServiceMsgProto.newBuilder().setAlarmSubUpdate(builder.build()).build()).build();
    return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg);
}

Example 10 with TbProtoQueueMsg

use of org.thingsboard.server.queue.common.TbProtoQueueMsg in project thingsboard by thingsboard.

the class DefaultSubscriptionManagerService method toProto.

private TbProtoQueueMsg<ToCoreNotificationMsg> toProto(TbSubscription subscription, List<TsKvEntry> updates, boolean ignoreEmptyUpdates) {
    TbSubscriptionUpdateProto.Builder builder = TbSubscriptionUpdateProto.newBuilder();
    builder.setSessionId(subscription.getSessionId());
    builder.setSubscriptionId(subscription.getSubscriptionId());
    Map<String, List<Object>> data = new TreeMap<>();
    for (TsKvEntry tsEntry : updates) {
        List<Object> values = data.computeIfAbsent(tsEntry.getKey(), k -> new ArrayList<>());
        Object[] value = new Object[2];
        value[0] = tsEntry.getTs();
        value[1] = tsEntry.getValueAsString();
        values.add(value);
    }
    data.forEach((key, value) -> {
        TbSubscriptionUpdateValueListProto.Builder dataBuilder = TbSubscriptionUpdateValueListProto.newBuilder();
        dataBuilder.setKey(key);
        boolean hasData = false;
        for (Object v : value) {
            Object[] array = (Object[]) v;
            TbSubscriptionUpdateTsValue.Builder tsValueBuilder = TbSubscriptionUpdateTsValue.newBuilder();
            tsValueBuilder.setTs((long) array[0]);
            String strVal = (String) array[1];
            if (strVal != null) {
                hasData = true;
                tsValueBuilder.setValue(strVal);
            }
            dataBuilder.addTsValue(tsValueBuilder.build());
        }
        if (!ignoreEmptyUpdates || hasData) {
            builder.addData(dataBuilder.build());
        }
    });
    ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg(LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build()).build();
    return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg);
}