Examples with NamespaceBundleStats com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats used on opensource projects

Example 11 with NamespaceBundleStats

use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.

the class SimpleLoadManagerImpl method doNamespaceBundleSplit.

/**
     * Detect and split hot namespace bundles
     */
@Override
public void doNamespaceBundleSplit() throws Exception {
    int maxBundleCount = pulsar.getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxBundleTopics = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    long maxBundleSessions = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxBundleMsgRate = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBundleBandwidth = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * MBytes;
    log.info("Running namespace bundle split with thresholds: topics {}, sessions {}, msgRate {}, bandwidth {}, maxBundles {}", maxBundleTopics, maxBundleSessions, maxBundleMsgRate, maxBundleBandwidth, maxBundleCount);
    if (this.lastLoadReport == null || this.lastLoadReport.getBundleStats() == null) {
        return;
    }
    Map<String, NamespaceBundleStats> bundleStats = this.lastLoadReport.getBundleStats();
    Set<String> bundlesToBeSplit = new HashSet<>();
    for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
        String bundleName = statsEntry.getKey();
        NamespaceBundleStats stats = statsEntry.getValue();
        long totalSessions = stats.consumerCount + stats.producerCount;
        double totalMsgRate = stats.msgRateIn + stats.msgRateOut;
        double totalBandwidth = stats.msgThroughputIn + stats.msgThroughputOut;
        boolean needSplit = false;
        if (stats.topics > maxBundleTopics || totalSessions > maxBundleSessions || totalMsgRate > maxBundleMsgRate || totalBandwidth > maxBundleBandwidth) {
            if (stats.topics <= 1) {
                log.info("Unable to split hot namespace bundle {} since there is only one topic.", bundleName);
            } else {
                NamespaceName namespaceName = new NamespaceName(getNamespaceNameFromBundleName(bundleName));
                int numBundles = pulsar.getNamespaceService().getBundleCount(namespaceName);
                if (numBundles >= maxBundleCount) {
                    log.info("Unable to split hot namespace bundle {} since the namespace has too many bundles.", bundleName);
                } else {
                    needSplit = true;
                }
            }
        }
        if (needSplit) {
            if (this.getLoadBalancerAutoBundleSplitEnabled()) {
                log.info("Will split hot namespace bundle {}, topics {}, producers+consumers {}, msgRate in+out {}, bandwidth in+out {}", bundleName, stats.topics, totalSessions, totalMsgRate, totalBandwidth);
                bundlesToBeSplit.add(bundleName);
            } else {
                log.info("DRY RUN - split hot namespace bundle {}, topics {}, producers+consumers {}, msgRate in+out {}, bandwidth in+out {}", bundleName, stats.topics, totalSessions, totalMsgRate, totalBandwidth);
            }
        }
    }
    if (bundlesToBeSplit.size() > 0) {
        for (String bundleName : bundlesToBeSplit) {
            try {
                pulsar.getAdminClient().namespaces().splitNamespaceBundle(getNamespaceNameFromBundleName(bundleName), getBundleRangeFromBundleName(bundleName));
                log.info("Successfully split namespace bundle {}", bundleName);
            } catch (Exception e) {
                log.error("Failed to split namespace bundle {}", bundleName, e);
            }
        }
        this.setLoadReportForceUpdateFlag();
    }
}