Examples with NamespaceBundleStats org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats used on opensource projects

Example 1 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class PulsarStats method updateStats.

public synchronized void updateStats(ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, Topic>>> topicsMap) {
    StatsOutputStream topicStatsStream = new StatsOutputStream(tempTopicStatsBuf);
    try {
        tempMetricsCollection.clear();
        bundleStats.clear();
        brokerOperabilityMetrics.reset();
        // Json begin
        topicStatsStream.startObject();
        topicsMap.forEach((namespaceName, bundles) -> {
            if (bundles.isEmpty()) {
                return;
            }
            try {
                topicStatsStream.startObject(namespaceName);
                nsStats.reset();
                bundles.forEach((bundle, topics) -> {
                    NamespaceBundleStats currentBundleStats = bundleStats.computeIfAbsent(bundle, k -> new NamespaceBundleStats());
                    currentBundleStats.reset();
                    currentBundleStats.topics = topics.size();
                    topicStatsStream.startObject(NamespaceBundle.getBundleRange(bundle));
                    tempNonPersistentTopics.clear();
                    // start persistent topic
                    topicStatsStream.startObject("persistent");
                    topics.forEach((name, topic) -> {
                        if (topic instanceof PersistentTopic) {
                            try {
                                topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                            } catch (Exception e) {
                                log.error("Failed to generate topic stats for topic {}: {}", name, e.getMessage(), e);
                            }
                            // this task: helps to activate inactive-backlog-cursors which have caught up and
                            // connected, also deactivate active-backlog-cursors which has backlog
                            ((PersistentTopic) topic).getManagedLedger().checkBackloggedCursors();
                        } else if (topic instanceof NonPersistentTopic) {
                            tempNonPersistentTopics.add((NonPersistentTopic) topic);
                        } else {
                            log.warn("Unsupported type of topic {}", topic.getClass().getName());
                        }
                    });
                    // end persistent topics section
                    topicStatsStream.endObject();
                    if (!tempNonPersistentTopics.isEmpty()) {
                        // start non-persistent topic
                        topicStatsStream.startObject("non-persistent");
                        tempNonPersistentTopics.forEach(topic -> {
                            try {
                                topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                            } catch (Exception e) {
                                log.error("Failed to generate topic stats for topic {}: {}", topic.getName(), e.getMessage(), e);
                            }
                        });
                        // end non-persistent topics section
                        topicStatsStream.endObject();
                    }
                    // end namespace-bundle section
                    topicStatsStream.endObject();
                });
                topicStatsStream.endObject();
                // Update metricsCollection with namespace stats
                tempMetricsCollection.add(nsStats.add(namespaceName));
            } catch (Exception e) {
                log.error("Failed to generate namespace stats for namespace {}: {}", namespaceName, e.getMessage(), e);
            }
        });
        if (clusterReplicationMetrics.isMetricsEnabled()) {
            clusterReplicationMetrics.get().forEach(clusterMetric -> tempMetricsCollection.add(clusterMetric));
            clusterReplicationMetrics.reset();
        }
        brokerOperabilityMetrics.getMetrics().forEach(brokerOperabilityMetric -> tempMetricsCollection.add(brokerOperabilityMetric));
        // json end
        topicStatsStream.endObject();
    } catch (Exception e) {
        log.error("Unable to update topic stats", e);
    }
    // swap metricsCollection and tempMetricsCollection
    List<Metrics> tempRefMetrics = metricsCollection;
    metricsCollection = tempMetricsCollection;
    tempMetricsCollection = tempRefMetrics;
    bufferLock.writeLock().lock();
    try {
        ByteBuf tmp = topicStatsBuf;
        topicStatsBuf = tempTopicStatsBuf;
        tempTopicStatsBuf = tmp;
        tempTopicStatsBuf.clear();
    } finally {
        bufferLock.writeLock().unlock();
    }
}

Example 2 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method updateBundleData.

// As the leader broker, use the local broker data saved on ZooKeeper to update the bundle stats so that better load
// management decisions may be made.
private void updateBundleData() {
    final Map<String, BundleData> bundleData = loadData.getBundleData();
    // Iterate over the broker data.
    for (Map.Entry<String, BrokerData> brokerEntry : loadData.getBrokerData().entrySet()) {
        final String broker = brokerEntry.getKey();
        final BrokerData brokerData = brokerEntry.getValue();
        final Map<String, NamespaceBundleStats> statsMap = brokerData.getLocalData().getLastStats();
        // broker to update the bundle data.
        for (Map.Entry<String, NamespaceBundleStats> entry : statsMap.entrySet()) {
            final String bundle = entry.getKey();
            final NamespaceBundleStats stats = entry.getValue();
            if (bundleData.containsKey(bundle)) {
                // If we recognize the bundle, add these stats as a new
                // sample.
                bundleData.get(bundle).update(stats);
            } else {
                // Otherwise, attempt to find the bundle data on ZooKeeper.
                // If it cannot be found, use the latest stats as the first
                // sample.
                BundleData currentBundleData = getBundleDataOrDefault(bundle);
                currentBundleData.update(stats);
                bundleData.put(bundle, currentBundleData);
            }
        }
        // Remove all loaded bundles from the preallocated maps.
        final Map<String, BundleData> preallocatedBundleData = brokerData.getPreallocatedBundleData();
        synchronized (preallocatedBundleData) {
            for (String preallocatedBundleName : brokerData.getPreallocatedBundleData().keySet()) {
                if (brokerData.getLocalData().getBundles().contains(preallocatedBundleName)) {
                    final Iterator<Map.Entry<String, BundleData>> preallocatedIterator = preallocatedBundleData.entrySet().iterator();
                    while (preallocatedIterator.hasNext()) {
                        final String bundle = preallocatedIterator.next().getKey();
                        if (bundleData.containsKey(bundle)) {
                            preallocatedIterator.remove();
                            preallocatedBundleToBroker.remove(bundle);
                        }
                    }
                }
                // This is needed too in case a broker which was assigned a bundle dies and comes back up.
                if (preallocatedBundleToBroker.containsKey(preallocatedBundleName)) {
                    preallocatedBundleToBroker.remove(preallocatedBundleName);
                }
            }
        }
        // Using the newest data, update the aggregated time-average data for the current broker.
        brokerData.getTimeAverageData().reset(statsMap.keySet(), bundleData, defaultStats);
        final Map<String, Set<String>> namespaceToBundleRange = brokerToNamespaceToBundleRange.computeIfAbsent(broker, k -> new HashMap<>());
        synchronized (namespaceToBundleRange) {
            namespaceToBundleRange.clear();
            LoadManagerShared.fillNamespaceToBundlesMap(statsMap.keySet(), namespaceToBundleRange);
            LoadManagerShared.fillNamespaceToBundlesMap(preallocatedBundleData.keySet(), namespaceToBundleRange);
        }
    }
}

Example 3 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method updateRealtimeResourceQuota.

private synchronized void updateRealtimeResourceQuota() {
    long memObjectGroupSize = 500;
    if (!currentLoadReports.isEmpty()) {
        long totalBundles = 0;
        long totalMemGroups = 0;
        double totalMsgRateIn = 0.0;
        double totalMsgRateOut = 0.0;
        double totalMsgRate = 0.0;
        double totalCpuUsage = 0.0;
        double totalMemoryUsage = 0.0;
        double totalBandwidthIn = 0.0;
        double totalBandwidthOut = 0.0;
        long loadReportTimestamp = -1;
        // update resource factors
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            LoadReport loadReport = entry.getValue();
            if (loadReport.getTimestamp() > loadReportTimestamp) {
                loadReportTimestamp = loadReport.getTimestamp();
            }
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                totalBundles++;
                NamespaceBundleStats stats = statsEntry.getValue();
                totalMemGroups += (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                totalBandwidthIn += stats.msgThroughputIn;
                totalBandwidthOut += stats.msgThroughputOut;
            }
            SystemResourceUsage resUsage = loadReport.getSystemResourceUsage();
            totalMsgRateIn += loadReport.getMsgRateIn();
            totalMsgRateOut += loadReport.getMsgRateOut();
            totalCpuUsage = totalCpuUsage + resUsage.getCpu().usage;
            totalMemoryUsage = totalMemoryUsage + resUsage.getMemory().usage;
        }
        totalMsgRate = totalMsgRateIn + totalMsgRateOut;
        long timePast = loadReportTimestamp - this.lastResourceQuotaUpdateTimestamp;
        this.lastResourceQuotaUpdateTimestamp = loadReportTimestamp;
        if (totalMsgRate > 1000 && totalMemGroups > 30) {
            this.realtimeCpuLoadFactor = timeSmoothValue(this.realtimeCpuLoadFactor, totalCpuUsage / totalMsgRate, RESOURCE_QUOTA_MIN_CPU_FACTOR, RESOURCE_QUOTA_MAX_CPU_FACTOR, timePast);
            this.realtimeMemoryLoadFactor = timeSmoothValue(this.realtimeMemoryLoadFactor, totalMemoryUsage / totalMemGroups, RESOURCE_QUOTA_MIN_MEM_FACTOR, RESOURCE_QUOTA_MAX_MEM_FACTOR, timePast);
        }
        // calculate average bundle
        if (totalBundles > 30 && this.realtimeAvgResourceQuota.getDynamic()) {
            ResourceQuota oldQuota = this.realtimeAvgResourceQuota;
            ResourceQuota newQuota = timeSmoothQuota(oldQuota, totalMsgRateIn / totalBundles, totalMsgRateOut / totalBundles, totalBandwidthIn / totalBundles, totalBandwidthOut / totalBundles, totalMemoryUsage / totalBundles, timePast);
            this.realtimeAvgResourceQuota = newQuota;
        }
        // update realtime quota for each bundle
        Map<String, ResourceQuota> newQuotas = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                String bundle = statsEntry.getKey();
                NamespaceBundleStats stats = statsEntry.getValue();
                long memGroupCount = (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                double newMemoryQuota = memGroupCount * this.realtimeMemoryLoadFactor;
                ResourceQuota oldQuota = getResourceQuota(bundle);
                ResourceQuota newQuota = timeSmoothQuota(oldQuota, stats.msgRateIn, stats.msgRateOut, stats.msgThroughputIn, stats.msgThroughputOut, newMemoryQuota, timePast);
                newQuotas.put(bundle, newQuota);
            }
        }
        this.realtimeResourceQuotas.set(newQuotas);
    }
}

Example 4 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testDeleteTopics.

@Test
public void testDeleteTopics() throws Exception {
    BrokerService brokerService = pulsar.getBrokerService();
    // 1. producers connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic("persistent://prop/use/ns-abc/topic-1").create();
    /* Producer<byte[]> producer2 = */
    pulsarClient.newProducer().topic("persistent://prop/use/ns-abc/topic-2").create();
    brokerService.updateRates();
    Map<String, NamespaceBundleStats> bundleStatsMap = brokerService.getBundleStats();
    assertEquals(bundleStatsMap.size(), 1);
    NamespaceBundleStats bundleStats = bundleStatsMap.get("prop/use/ns-abc/0x00000000_0xffffffff");
    assertNotNull(bundleStats);
    producer1.close();
    admin.persistentTopics().delete("persistent://prop/use/ns-abc/topic-1");
    brokerService.updateRates();
    bundleStatsMap = brokerService.getBundleStats();
    assertEquals(bundleStatsMap.size(), 1);
    bundleStats = bundleStatsMap.get("prop/use/ns-abc/0x00000000_0xffffffff");
    assertNotNull(bundleStats);
// // Delete 2nd topic as well
// producer2.close();
// admin.persistentTopics().delete("persistent://prop/use/ns-abc/topic-2");
// 
// brokerService.updateRates();
// 
// bundleStatsMap = brokerService.getBundleStats();
// assertEquals(bundleStatsMap.size(), 0);
}

Example 5 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class LoadBalancerTest method testNamespaceBundleAutoSplit.

/**
 * Test the namespace bundle auto-split
 */
@Test
public void testNamespaceBundleAutoSplit() throws Exception {
    int maxBundles = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxTopics = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    int maxSessions = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxMsgRate = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBandwidth = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1048576;
    pulsarServices[0].getConfiguration().setLoadBalancerAutoBundleSplitEnabled(true);
    // create namespaces
    for (int i = 1; i <= 10; i++) {
        int numBundles = (i == 10) ? maxBundles : 2;
        createNamespace(pulsarServices[0], String.format("pulsar/use/primary-ns-%02d", i), numBundles);
    }
    // fake Namespaces Admin
    NamespacesImpl namespaceAdmin = mock(NamespacesImpl.class);
    setObjectField(PulsarAdmin.class, pulsarServices[0].getAdminClient(), "namespaces", namespaceAdmin);
    // create load report
    // namespace 01~09 need to be split
    // namespace 08~10 don't need or cannot be split
    LoadReport lr = new LoadReport();
    lr.setName(lookupAddresses[0]);
    lr.setSystemResourceUsage(new SystemResourceUsage());
    Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
    bundleStats.put("pulsar/use/primary-ns-01/0x00000000_0x80000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-02/0x00000000_0x80000000", newBundleStats(2, maxSessions + 1, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-03/0x00000000_0x80000000", newBundleStats(2, 0, maxSessions + 1, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-04/0x00000000_0x80000000", newBundleStats(2, 0, 0, maxMsgRate + 1, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-05/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, maxMsgRate + 1, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-06/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, maxBandwidth + 1, 0));
    bundleStats.put("pulsar/use/primary-ns-07/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-08/0x00000000_0x80000000", newBundleStats(maxTopics - 1, maxSessions - 1, 1, maxMsgRate - 1, 1, maxBandwidth - 1, 1));
    bundleStats.put("pulsar/use/primary-ns-09/0x00000000_0x80000000", newBundleStats(1, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-10/0x00000000_0x02000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    lr.setBundleStats(bundleStats);
    setObjectField(SimpleLoadManagerImpl.class, pulsarServices[0].getLoadManager().get(), "lastLoadReport", lr);
    String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[0]);
    String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
    bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    // sleep to wait load ranking be triggered and trigger bundle split
    Thread.sleep(5000);
    pulsarServices[0].getLoadManager().get().doNamespaceBundleSplit();
    boolean isAutoUnooadSplitBundleEnabled = pulsarServices[0].getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled();
    // verify bundles are split
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-01", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-02", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-03", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-04", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-05", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-06", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-07", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-08", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-09", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-10", "0x00000000_0x02000000", isAutoUnooadSplitBundleEnabled);
}