Examples with TimeAverageMessageData org.apache.pulsar.broker.TimeAverageMessageData used on opensource projects

Example 6 with TimeAverageMessageData

use of org.apache.pulsar.broker.TimeAverageMessageData in project incubator-pulsar by apache.

the class BundleSplitterTask method findBundlesToSplit.

/**
 * Determines which bundles should be split based on various thresholds.
 *
 * @param loadData
 *            Load data to base decisions on (does not have benefit of preallocated data since this may not be the
 *            leader broker).
 * @param localData
 *            Local data for the broker we are splitting on.
 * @param pulsar
 *            Service to use.
 * @return All bundles who have exceeded configured thresholds in number of topics, number of sessions, total
 *         message rates, or total throughput.
 */
@Override
public Set<String> findBundlesToSplit(final LoadData loadData, final PulsarService pulsar) {
    bundleCache.clear();
    final ServiceConfiguration conf = pulsar.getConfiguration();
    int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles();
    long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
    long maxBundleSessions = conf.getLoadBalancerNamespaceBundleMaxSessions();
    long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
    loadData.getBrokerData().forEach((broker, brokerData) -> {
        LocalBrokerData localData = brokerData.getLocalData();
        for (final Map.Entry<String, NamespaceBundleStats> entry : localData.getLastStats().entrySet()) {
            final String bundle = entry.getKey();
            final NamespaceBundleStats stats = entry.getValue();
            double totalMessageRate = 0;
            double totalMessageThroughput = 0;
            // Attempt to consider long-term message data, otherwise effectively ignore.
            if (loadData.getBundleData().containsKey(bundle)) {
                final TimeAverageMessageData longTermData = loadData.getBundleData().get(bundle).getLongTermData();
                totalMessageRate = longTermData.totalMsgRate();
                totalMessageThroughput = longTermData.totalMsgThroughput();
            }
            if (stats.topics > maxBundleTopics || stats.consumerCount + stats.producerCount > maxBundleSessions || totalMessageRate > maxBundleMsgRate || totalMessageThroughput > maxBundleBandwidth) {
                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
                try {
                    final int bundleCount = pulsar.getNamespaceService().getBundleCount(NamespaceName.get(namespace));
                    if (bundleCount < maxBundleCount) {
                        bundleCache.add(bundle);
                    } else {
                        log.warn("Could not split namespace bundle {} because namespace {} has too many bundles: {}", bundle, namespace, bundleCount);
                    }
                } catch (Exception e) {
                    log.warn("Error while getting bundle count for namespace {}", namespace, e);
                }
            }
        }
    });
    return bundleCache;
}