Search in sources :

Example 1 with ServiceQueueKey

use of org.thingsboard.server.common.msg.queue.ServiceQueueKey in project thingsboard by thingsboard.

the class HashPartitionService method getServiceKeyListMap.

private Map<ServiceQueueKey, List<ServiceInfo>> getServiceKeyListMap(List<ServiceInfo> services) {
    final Map<ServiceQueueKey, List<ServiceInfo>> currentMap = new HashMap<>();
    services.forEach(serviceInfo -> {
        for (String serviceTypeStr : serviceInfo.getServiceTypesList()) {
            ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase());
            if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) {
                for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) {
                    ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo));
                    currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo);
                }
            } else {
                ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo));
                currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo);
            }
        }
    });
    return currentMap;
}
Also used : ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) ClusterTopologyChangeEvent(org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) QueueService(org.thingsboard.server.queue.QueueService) TenantId(org.thingsboard.server.common.data.id.TenantId) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) Service(org.springframework.stereotype.Service) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) Map(java.util.Map) ApplicationEventPublisher(org.springframework.context.ApplicationEventPublisher) EntityId(org.thingsboard.server.common.data.id.EntityId) Hasher(com.google.common.hash.Hasher) TbQueueRuleEngineSettings(org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings) ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) ServiceListChangedEvent(org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent) PartitionChangeEvent(org.thingsboard.server.queue.discovery.event.PartitionChangeEvent) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) List(java.util.List) PostConstruct(javax.annotation.PostConstruct) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos) HashFunction(com.google.common.hash.HashFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) StringUtils(org.springframework.util.StringUtils) TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos)

Example 2 with ServiceQueueKey

use of org.thingsboard.server.common.msg.queue.ServiceQueueKey in project thingsboard by thingsboard.

the class HashPartitionService method addNode.

private void addNode(Map<ServiceQueueKey, List<ServiceInfo>> queueServiceList, ServiceInfo instance) {
    TenantId tenantId = getSystemOrIsolatedTenantId(instance);
    for (String serviceTypeStr : instance.getServiceTypesList()) {
        ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase());
        if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) {
            for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) {
                ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId);
                partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions());
                partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic());
                queueServiceList.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(instance);
            }
        } else {
            ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId);
            queueServiceList.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(instance);
        }
    }
    for (String transportType : instance.getTransportsList()) {
        tbTransportServicesByType.computeIfAbsent(transportType, t -> new ArrayList<>()).add(instance);
    }
}
Also used : ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) ClusterTopologyChangeEvent(org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) QueueService(org.thingsboard.server.queue.QueueService) TenantId(org.thingsboard.server.common.data.id.TenantId) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) Service(org.springframework.stereotype.Service) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) Map(java.util.Map) ApplicationEventPublisher(org.springframework.context.ApplicationEventPublisher) EntityId(org.thingsboard.server.common.data.id.EntityId) Hasher(com.google.common.hash.Hasher) TbQueueRuleEngineSettings(org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings) ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) ServiceListChangedEvent(org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent) PartitionChangeEvent(org.thingsboard.server.queue.discovery.event.PartitionChangeEvent) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) List(java.util.List) PostConstruct(javax.annotation.PostConstruct) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos) HashFunction(com.google.common.hash.HashFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) StringUtils(org.springframework.util.StringUtils) TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) TenantId(org.thingsboard.server.common.data.id.TenantId) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) ArrayList(java.util.ArrayList) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos)

Example 3 with ServiceQueueKey

use of org.thingsboard.server.common.msg.queue.ServiceQueueKey in project thingsboard by thingsboard.

the class HashPartitionService method recalculatePartitions.

@Override
public synchronized void recalculatePartitions(ServiceInfo currentService, List<ServiceInfo> otherServices) {
    tbTransportServicesByType.clear();
    logServiceInfo(currentService);
    otherServices.forEach(this::logServiceInfo);
    Map<ServiceQueueKey, List<ServiceInfo>> queueServicesMap = new HashMap<>();
    addNode(queueServicesMap, currentService);
    for (ServiceInfo other : otherServices) {
        addNode(queueServicesMap, other);
    }
    queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId)));
    ConcurrentMap<ServiceQueueKey, List<Integer>> oldPartitions = myPartitions;
    TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService);
    myPartitions = new ConcurrentHashMap<>();
    partitionSizes.forEach((serviceQueue, size) -> {
        ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(serviceQueue, myIsolatedOrSystemTenantId);
        for (int i = 0; i < size; i++) {
            ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(myServiceQueueKey), i);
            if (currentService.equals(serviceInfo)) {
                ServiceQueueKey serviceQueueKey = new ServiceQueueKey(serviceQueue, getSystemOrIsolatedTenantId(serviceInfo));
                myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i);
            }
        }
    });
    tpiCache.clear();
    oldPartitions.forEach((serviceQueueKey, partitions) -> {
        if (!myPartitions.containsKey(serviceQueueKey)) {
            log.info("[{}] NO MORE PARTITIONS FOR CURRENT KEY", serviceQueueKey);
            applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, Collections.emptySet()));
        }
    });
    myPartitions.forEach((serviceQueueKey, partitions) -> {
        if (!partitions.equals(oldPartitions.get(serviceQueueKey))) {
            log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions);
            Set<TopicPartitionInfo> tpiList = partitions.stream().map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition)).collect(Collectors.toSet());
            applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList));
        }
    });
    if (currentOtherServices == null) {
        currentOtherServices = new ArrayList<>(otherServices);
    } else {
        Set<ServiceQueueKey> changes = new HashSet<>();
        Map<ServiceQueueKey, List<ServiceInfo>> currentMap = getServiceKeyListMap(currentOtherServices);
        Map<ServiceQueueKey, List<ServiceInfo>> newMap = getServiceKeyListMap(otherServices);
        currentOtherServices = otherServices;
        currentMap.forEach((key, list) -> {
            if (!list.equals(newMap.get(key))) {
                changes.add(key);
            }
        });
        currentMap.keySet().forEach(newMap::remove);
        changes.addAll(newMap.keySet());
        if (!changes.isEmpty()) {
            applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes));
        }
    }
    applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService));
}
Also used : ClusterTopologyChangeEvent(org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) QueueService(org.thingsboard.server.queue.QueueService) TenantId(org.thingsboard.server.common.data.id.TenantId) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) Service(org.springframework.stereotype.Service) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) Map(java.util.Map) ApplicationEventPublisher(org.springframework.context.ApplicationEventPublisher) EntityId(org.thingsboard.server.common.data.id.EntityId) Hasher(com.google.common.hash.Hasher) TbQueueRuleEngineSettings(org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings) ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) ServiceListChangedEvent(org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent) PartitionChangeEvent(org.thingsboard.server.queue.discovery.event.PartitionChangeEvent) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) List(java.util.List) PostConstruct(javax.annotation.PostConstruct) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos) HashFunction(com.google.common.hash.HashFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) StringUtils(org.springframework.util.StringUtils) TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterTopologyChangeEvent(org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) ArrayList(java.util.ArrayList) PartitionChangeEvent(org.thingsboard.server.queue.discovery.event.PartitionChangeEvent) ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) TenantId(org.thingsboard.server.common.data.id.TenantId) TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) ArrayList(java.util.ArrayList) List(java.util.List) ServiceListChangedEvent(org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent) HashSet(java.util.HashSet)

Example 4 with ServiceQueueKey

use of org.thingsboard.server.common.msg.queue.ServiceQueueKey in project thingsboard by thingsboard.

the class HashPartitionService method buildTopicPartitionInfo.

private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueue serviceQueue, TenantId tenantId, int partition) {
    TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder();
    tpi.topic(partitionTopics.get(serviceQueue));
    tpi.partition(partition);
    ServiceQueueKey myPartitionsSearchKey;
    if (isIsolated(serviceQueue, tenantId)) {
        tpi.tenantId(tenantId);
        myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, tenantId);
    } else {
        myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, TenantId.SYS_TENANT_ID);
    }
    List<Integer> partitions = myPartitions.get(myPartitionsSearchKey);
    if (partitions != null) {
        tpi.myPartition(partitions.contains(partition));
    } else {
        tpi.myPartition(false);
    }
    return tpi.build();
}
Also used : TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey)

Aggregations

ServiceQueueKey (org.thingsboard.server.common.msg.queue.ServiceQueueKey)4 TopicPartitionInfo (org.thingsboard.server.common.msg.queue.TopicPartitionInfo)4 HashFunction (com.google.common.hash.HashFunction)3 Hasher (com.google.common.hash.Hasher)3 Hashing (com.google.common.hash.Hashing)3 ArrayList (java.util.ArrayList)3 Collections (java.util.Collections)3 Comparator (java.util.Comparator)3 HashMap (java.util.HashMap)3 HashSet (java.util.HashSet)3 List (java.util.List)3 Map (java.util.Map)3 Set (java.util.Set)3 UUID (java.util.UUID)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 ConcurrentMap (java.util.concurrent.ConcurrentMap)3 Collectors (java.util.stream.Collectors)3 PostConstruct (javax.annotation.PostConstruct)3 Slf4j (lombok.extern.slf4j.Slf4j)3 Value (org.springframework.beans.factory.annotation.Value)3