Examples with TimeAverageMessageData org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData used on opensource projects

Example 1 with TimeAverageMessageData

use of org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData in project pulsar by yahoo.

the class OverloadShedderTest method testBrokerWithMultipleBundles.

@Test
public void testBrokerWithMultipleBundles() {
    int numBundles = 10;
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(999, 1000));
    broker1.setBandwidthOut(new ResourceUsage(999, 1000));
    LocalBrokerData anotherBroker = new LocalBrokerData();
    String anotherBrokerName = "another-broker";
    double brokerThroghput = 0;
    for (int i = 1; i <= numBundles; i++) {
        broker1.getBundles().add("bundle-" + i);
        BundleData bundle = new BundleData();
        TimeAverageMessageData db = new TimeAverageMessageData();
        double throughput = i * 1024 * 1024;
        db.setMsgThroughputIn(throughput);
        db.setMsgThroughputOut(throughput);
        bundle.setShortTermData(db);
        loadData.getBundleData().put("bundle-" + i, bundle);
        // This bundle should not be selected for `broker1` since it is belong to another broker.
        String anotherBundleName = anotherBrokerName + "-bundle-" + (numBundles + i);
        loadData.getBundleData().put(anotherBundleName, bundle);
        anotherBroker.getBundles().add(anotherBundleName);
        brokerThroghput += throughput;
    }
    broker1.setMsgThroughputIn(brokerThroghput);
    broker1.setMsgThroughputOut(brokerThroghput);
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    loadData.getBrokerData().put(anotherBrokerName, new BrokerData(anotherBroker));
    Multimap<String, String> bundlesToUnload = os.findBundlesForUnloading(loadData, conf);
    assertFalse(bundlesToUnload.isEmpty());
    assertEquals(bundlesToUnload.get("broker-1"), Lists.newArrayList("bundle-10", "bundle-9"));
}

Example 2 with TimeAverageMessageData

use of org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData in project pulsar by yahoo.

the class OverloadShedderTest method testBrokerWithSingleBundle.

@Test
public void testBrokerWithSingleBundle() {
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(999, 1000));
    broker1.setBandwidthOut(new ResourceUsage(999, 1000));
    broker1.setBundles(Sets.newHashSet("bundle-1"));
    BundleData bundle1 = new BundleData();
    TimeAverageMessageData db1 = new TimeAverageMessageData();
    db1.setMsgThroughputIn(1000);
    db1.setMsgThroughputOut(1000);
    bundle1.setShortTermData(db1);
    loadData.getBundleData().put("bundle-1", bundle1);
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    assertTrue(os.findBundlesForUnloading(loadData, conf).isEmpty());
}

Example 3 with TimeAverageMessageData

use of org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData in project pulsar by yahoo.

the class OverloadShedderTest method testFilterRecentlyUnloaded.

@Test
public void testFilterRecentlyUnloaded() {
    int numBundles = 10;
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(999, 1000));
    broker1.setBandwidthOut(new ResourceUsage(999, 1000));
    double brokerThroghput = 0;
    for (int i = 1; i <= numBundles; i++) {
        broker1.getBundles().add("bundle-" + i);
        BundleData bundle = new BundleData();
        TimeAverageMessageData db = new TimeAverageMessageData();
        double throughput = i * 1024 * 1024;
        db.setMsgThroughputIn(throughput);
        db.setMsgThroughputOut(throughput);
        bundle.setShortTermData(db);
        loadData.getBundleData().put("bundle-" + i, bundle);
        brokerThroghput += throughput;
    }
    broker1.setMsgThroughputIn(brokerThroghput);
    broker1.setMsgThroughputOut(brokerThroghput);
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    loadData.getRecentlyUnloadedBundles().put("bundle-10", 1L);
    loadData.getRecentlyUnloadedBundles().put("bundle-9", 1L);
    Multimap<String, String> bundlesToUnload = os.findBundlesForUnloading(loadData, conf);
    assertFalse(bundlesToUnload.isEmpty());
    assertEquals(bundlesToUnload.get("broker-1"), Lists.newArrayList("bundle-8", "bundle-7"));
}

Example 4 with TimeAverageMessageData

use of org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData in project pulsar by yahoo.

the class ThresholdShedderTest method testBrokerNotReachThreshold.

@Test
public void testBrokerNotReachThreshold() {
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(500, 1000));
    broker1.setBandwidthOut(new ResourceUsage(500, 1000));
    broker1.setBundles(Sets.newHashSet("bundle-1"));
    BundleData bundleData = new BundleData();
    TimeAverageMessageData timeAverageMessageData = new TimeAverageMessageData();
    timeAverageMessageData.setMsgThroughputIn(1000);
    timeAverageMessageData.setMsgThroughputOut(1000);
    bundleData.setShortTermData(timeAverageMessageData);
    loadData.getBundleData().put("bundle-1", bundleData);
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    assertTrue(thresholdShedder.findBundlesForUnloading(loadData, conf).isEmpty());
}

Example 5 with TimeAverageMessageData

use of org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData in project pulsar by yahoo.

the class BundleSplitterTask method findBundlesToSplit.

/**
 * Determines which bundles should be split based on various thresholds.
 *
 * @param loadData
 *            Load data to base decisions on (does not have benefit of preallocated data since this may not be the
 *            leader broker).
 * @param pulsar
 *            Service to use.
 * @return All bundles who have exceeded configured thresholds in number of topics, number of sessions, total
 *         message rates, or total throughput.
 */
@Override
public Set<String> findBundlesToSplit(final LoadData loadData, final PulsarService pulsar) {
    bundleCache.clear();
    final ServiceConfiguration conf = pulsar.getConfiguration();
    int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles();
    long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
    long maxBundleSessions = conf.getLoadBalancerNamespaceBundleMaxSessions();
    long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
    loadData.getBrokerData().forEach((broker, brokerData) -> {
        LocalBrokerData localData = brokerData.getLocalData();
        for (final Map.Entry<String, NamespaceBundleStats> entry : localData.getLastStats().entrySet()) {
            final String bundle = entry.getKey();
            final NamespaceBundleStats stats = entry.getValue();
            if (stats.topics < 2) {
                log.info("The count of topics on the bundle {} is less than 2, skip split!", bundle);
                continue;
            }
            double totalMessageRate = 0;
            double totalMessageThroughput = 0;
            // Attempt to consider long-term message data, otherwise effectively ignore.
            if (loadData.getBundleData().containsKey(bundle)) {
                final TimeAverageMessageData longTermData = loadData.getBundleData().get(bundle).getLongTermData();
                totalMessageRate = longTermData.totalMsgRate();
                totalMessageThroughput = longTermData.totalMsgThroughput();
            }
            if (stats.topics > maxBundleTopics || (maxBundleSessions > 0 && (stats.consumerCount + stats.producerCount > maxBundleSessions)) || totalMessageRate > maxBundleMsgRate || totalMessageThroughput > maxBundleBandwidth) {
                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
                try {
                    final int bundleCount = pulsar.getNamespaceService().getBundleCount(NamespaceName.get(namespace));
                    if (bundleCount < maxBundleCount) {
                        bundleCache.add(bundle);
                    } else {
                        if (log.isDebugEnabled()) {
                            log.debug("Could not split namespace bundle {} because namespace {} has too many bundles:" + "{}", bundle, namespace, bundleCount);
                        }
                    }
                } catch (Exception e) {
                    log.warn("Error while getting bundle count for namespace {}", namespace, e);
                }
            }
        }
    });
    return bundleCache;
}