Search in sources :

Example 1 with ServiceInfo

use of org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo in project thingsboard by thingsboard.

the class SnmpTransportBalancingService method recalculatePartitions.

private void recalculatePartitions(List<ServiceInfo> otherServices, ServiceInfo currentService) {
    log.info("Recalculating partitions for SNMP transports");
    List<ServiceInfo> snmpTransports = Stream.concat(otherServices.stream(), Stream.of(currentService)).filter(service -> service.getTransportsList().contains(snmpTransportService.getName())).sorted(Comparator.comparing(ServiceInfo::getServiceId)).collect(Collectors.toList());
    log.trace("Found SNMP transports: {}", snmpTransports);
    int previousCurrentTransportPartitionIndex = currentTransportPartitionIndex;
    int previousSnmpTransportsCount = snmpTransportsCount;
    if (!snmpTransports.isEmpty()) {
        for (int i = 0; i < snmpTransports.size(); i++) {
            if (snmpTransports.get(i).equals(currentService)) {
                currentTransportPartitionIndex = i;
                break;
            }
        }
        snmpTransportsCount = snmpTransports.size();
    }
    if (snmpTransportsCount != previousSnmpTransportsCount || currentTransportPartitionIndex != previousCurrentTransportPartitionIndex) {
        log.info("SNMP transports partitions have changed: transports count = {}, current transport partition index = {}", snmpTransportsCount, currentTransportPartitionIndex);
        eventPublisher.publishEvent(new SnmpTransportListChangedEvent());
    } else {
        log.info("SNMP transports partitions have not changed");
    }
}
Also used : ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) SnmpTransportListChangedEvent(org.thingsboard.server.transport.snmp.event.SnmpTransportListChangedEvent)

Example 2 with ServiceInfo

use of org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo in project thingsboard by thingsboard.

the class HashPartitionService method getServiceKeyListMap.

private Map<ServiceQueueKey, List<ServiceInfo>> getServiceKeyListMap(List<ServiceInfo> services) {
    final Map<ServiceQueueKey, List<ServiceInfo>> currentMap = new HashMap<>();
    services.forEach(serviceInfo -> {
        for (String serviceTypeStr : serviceInfo.getServiceTypesList()) {
            ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase());
            if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) {
                for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) {
                    ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo));
                    currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo);
                }
            } else {
                ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo));
                currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo);
            }
        }
    });
    return currentMap;
}
Also used : ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) ClusterTopologyChangeEvent(org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) QueueService(org.thingsboard.server.queue.QueueService) TenantId(org.thingsboard.server.common.data.id.TenantId) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) Service(org.springframework.stereotype.Service) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) Map(java.util.Map) ApplicationEventPublisher(org.springframework.context.ApplicationEventPublisher) EntityId(org.thingsboard.server.common.data.id.EntityId) Hasher(com.google.common.hash.Hasher) TbQueueRuleEngineSettings(org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings) ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) ServiceListChangedEvent(org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent) PartitionChangeEvent(org.thingsboard.server.queue.discovery.event.PartitionChangeEvent) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) List(java.util.List) PostConstruct(javax.annotation.PostConstruct) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos) HashFunction(com.google.common.hash.HashFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) StringUtils(org.springframework.util.StringUtils) TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos)

Example 3 with ServiceInfo

use of org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo in project thingsboard by thingsboard.

the class HashPartitionService method addNode.

private void addNode(Map<ServiceQueueKey, List<ServiceInfo>> queueServiceList, ServiceInfo instance) {
    TenantId tenantId = getSystemOrIsolatedTenantId(instance);
    for (String serviceTypeStr : instance.getServiceTypesList()) {
        ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase());
        if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) {
            for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) {
                ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId);
                partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions());
                partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic());
                queueServiceList.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(instance);
            }
        } else {
            ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId);
            queueServiceList.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(instance);
        }
    }
    for (String transportType : instance.getTransportsList()) {
        tbTransportServicesByType.computeIfAbsent(transportType, t -> new ArrayList<>()).add(instance);
    }
}
Also used : ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) ClusterTopologyChangeEvent(org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) QueueService(org.thingsboard.server.queue.QueueService) TenantId(org.thingsboard.server.common.data.id.TenantId) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) Service(org.springframework.stereotype.Service) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) Map(java.util.Map) ApplicationEventPublisher(org.springframework.context.ApplicationEventPublisher) EntityId(org.thingsboard.server.common.data.id.EntityId) Hasher(com.google.common.hash.Hasher) TbQueueRuleEngineSettings(org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings) ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) ServiceListChangedEvent(org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent) PartitionChangeEvent(org.thingsboard.server.queue.discovery.event.PartitionChangeEvent) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) List(java.util.List) PostConstruct(javax.annotation.PostConstruct) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos) HashFunction(com.google.common.hash.HashFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) StringUtils(org.springframework.util.StringUtils) TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) TenantId(org.thingsboard.server.common.data.id.TenantId) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) ArrayList(java.util.ArrayList) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos)

Example 4 with ServiceInfo

use of org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo in project thingsboard by thingsboard.

the class HashPartitionService method getAllServiceIds.

@Override
public Set<String> getAllServiceIds(ServiceType serviceType) {
    Set<String> result = new HashSet<>();
    ServiceInfo current = serviceInfoProvider.getServiceInfo();
    if (current.getServiceTypesList().contains(serviceType.name())) {
        result.add(current.getServiceId());
    }
    if (currentOtherServices != null) {
        for (ServiceInfo serviceInfo : currentOtherServices) {
            if (serviceInfo.getServiceTypesList().contains(serviceType.name())) {
                result.add(serviceInfo.getServiceId());
            }
        }
    }
    return result;
}
Also used : ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) HashSet(java.util.HashSet)

Example 5 with ServiceInfo

use of org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo in project thingsboard by thingsboard.

the class HashPartitionService method recalculatePartitions.

@Override
public synchronized void recalculatePartitions(ServiceInfo currentService, List<ServiceInfo> otherServices) {
    tbTransportServicesByType.clear();
    logServiceInfo(currentService);
    otherServices.forEach(this::logServiceInfo);
    Map<ServiceQueueKey, List<ServiceInfo>> queueServicesMap = new HashMap<>();
    addNode(queueServicesMap, currentService);
    for (ServiceInfo other : otherServices) {
        addNode(queueServicesMap, other);
    }
    queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId)));
    ConcurrentMap<ServiceQueueKey, List<Integer>> oldPartitions = myPartitions;
    TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService);
    myPartitions = new ConcurrentHashMap<>();
    partitionSizes.forEach((serviceQueue, size) -> {
        ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(serviceQueue, myIsolatedOrSystemTenantId);
        for (int i = 0; i < size; i++) {
            ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(myServiceQueueKey), i);
            if (currentService.equals(serviceInfo)) {
                ServiceQueueKey serviceQueueKey = new ServiceQueueKey(serviceQueue, getSystemOrIsolatedTenantId(serviceInfo));
                myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i);
            }
        }
    });
    tpiCache.clear();
    oldPartitions.forEach((serviceQueueKey, partitions) -> {
        if (!myPartitions.containsKey(serviceQueueKey)) {
            log.info("[{}] NO MORE PARTITIONS FOR CURRENT KEY", serviceQueueKey);
            applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, Collections.emptySet()));
        }
    });
    myPartitions.forEach((serviceQueueKey, partitions) -> {
        if (!partitions.equals(oldPartitions.get(serviceQueueKey))) {
            log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions);
            Set<TopicPartitionInfo> tpiList = partitions.stream().map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition)).collect(Collectors.toSet());
            applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList));
        }
    });
    if (currentOtherServices == null) {
        currentOtherServices = new ArrayList<>(otherServices);
    } else {
        Set<ServiceQueueKey> changes = new HashSet<>();
        Map<ServiceQueueKey, List<ServiceInfo>> currentMap = getServiceKeyListMap(currentOtherServices);
        Map<ServiceQueueKey, List<ServiceInfo>> newMap = getServiceKeyListMap(otherServices);
        currentOtherServices = otherServices;
        currentMap.forEach((key, list) -> {
            if (!list.equals(newMap.get(key))) {
                changes.add(key);
            }
        });
        currentMap.keySet().forEach(newMap::remove);
        changes.addAll(newMap.keySet());
        if (!changes.isEmpty()) {
            applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes));
        }
    }
    applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService));
}
Also used : ClusterTopologyChangeEvent(org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) QueueService(org.thingsboard.server.queue.QueueService) TenantId(org.thingsboard.server.common.data.id.TenantId) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) ServiceType(org.thingsboard.server.common.msg.queue.ServiceType) Service(org.springframework.stereotype.Service) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) Map(java.util.Map) ApplicationEventPublisher(org.springframework.context.ApplicationEventPublisher) EntityId(org.thingsboard.server.common.data.id.EntityId) Hasher(com.google.common.hash.Hasher) TbQueueRuleEngineSettings(org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings) ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) ServiceListChangedEvent(org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent) PartitionChangeEvent(org.thingsboard.server.queue.discovery.event.PartitionChangeEvent) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Slf4j(lombok.extern.slf4j.Slf4j) ServiceQueue(org.thingsboard.server.common.msg.queue.ServiceQueue) List(java.util.List) PostConstruct(javax.annotation.PostConstruct) TransportProtos(org.thingsboard.server.gen.transport.TransportProtos) HashFunction(com.google.common.hash.HashFunction) Comparator(java.util.Comparator) Collections(java.util.Collections) StringUtils(org.springframework.util.StringUtils) TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterTopologyChangeEvent(org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent) ServiceQueueKey(org.thingsboard.server.common.msg.queue.ServiceQueueKey) ArrayList(java.util.ArrayList) PartitionChangeEvent(org.thingsboard.server.queue.discovery.event.PartitionChangeEvent) ServiceInfo(org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo) TenantId(org.thingsboard.server.common.data.id.TenantId) TopicPartitionInfo(org.thingsboard.server.common.msg.queue.TopicPartitionInfo) ArrayList(java.util.ArrayList) List(java.util.List) ServiceListChangedEvent(org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent) HashSet(java.util.HashSet)

Aggregations

ServiceInfo (org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo)6 HashSet (java.util.HashSet)4 UUID (java.util.UUID)4 PostConstruct (javax.annotation.PostConstruct)4 ServiceType (org.thingsboard.server.common.msg.queue.ServiceType)4 TransportProtos (org.thingsboard.server.gen.transport.TransportProtos)4 HashFunction (com.google.common.hash.HashFunction)3 Hasher (com.google.common.hash.Hasher)3 Hashing (com.google.common.hash.Hashing)3 ArrayList (java.util.ArrayList)3 Collections (java.util.Collections)3 Comparator (java.util.Comparator)3 HashMap (java.util.HashMap)3 List (java.util.List)3 Map (java.util.Map)3 Set (java.util.Set)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 ConcurrentMap (java.util.concurrent.ConcurrentMap)3 Collectors (java.util.stream.Collectors)3 Slf4j (lombok.extern.slf4j.Slf4j)3