Examples with NamespaceBundleStats com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats used on opensource projects

Example 6 with NamespaceBundleStats

use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.

the class LoadBalancerTest method testNamespaceBundleAutoSplit.

/**
     * Test the namespace bundle auto-split
     */
@Test
public void testNamespaceBundleAutoSplit() throws Exception {
    int maxBundles = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxTopics = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    int maxSessions = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxMsgRate = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBandwidth = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1048576;
    pulsarServices[0].getConfiguration().setLoadBalancerAutoBundleSplitEnabled(true);
    // create namespaces
    for (int i = 1; i <= 10; i++) {
        int numBundles = (i == 10) ? maxBundles : 2;
        createNamespace(pulsarServices[0], String.format("pulsar/use/primary-ns-%02d", i), numBundles);
    }
    // fake Namespaces Admin
    NamespacesImpl namespaceAdmin = mock(NamespacesImpl.class);
    setObjectField(PulsarAdmin.class, pulsarServices[0].getAdminClient(), "namespaces", namespaceAdmin);
    // create load report
    // namespace 01~09 need to be split
    // namespace 08~10 don't need or cannot be split
    LoadReport lr = new LoadReport();
    lr.setName(lookupAddresses[0]);
    lr.setSystemResourceUsage(new SystemResourceUsage());
    Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
    bundleStats.put("pulsar/use/primary-ns-01/0x00000000_0x80000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-02/0x00000000_0x80000000", newBundleStats(2, maxSessions + 1, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-03/0x00000000_0x80000000", newBundleStats(2, 0, maxSessions + 1, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-04/0x00000000_0x80000000", newBundleStats(2, 0, 0, maxMsgRate + 1, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-05/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, maxMsgRate + 1, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-06/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, maxBandwidth + 1, 0));
    bundleStats.put("pulsar/use/primary-ns-07/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-08/0x00000000_0x80000000", newBundleStats(maxTopics - 1, maxSessions - 1, 1, maxMsgRate - 1, 1, maxBandwidth - 1, 1));
    bundleStats.put("pulsar/use/primary-ns-09/0x00000000_0x80000000", newBundleStats(1, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-10/0x00000000_0x02000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    lr.setBundleStats(bundleStats);
    setObjectField(SimpleLoadManagerImpl.class, pulsarServices[0].getLoadManager(), "lastLoadReport", lr);
    String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[0]);
    String loadReportJson = objectMapper.writeValueAsString(lr);
    bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    // sleep to wait load ranking be triggered and trigger bundle split
    Thread.sleep(5000);
    pulsarServices[0].getLoadManager().doNamespaceBundleSplit();
    // verify bundles are split
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-01", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-02", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-03", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-04", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-05", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-06", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-07", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-08", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-09", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-10", "0x00000000_0x02000000");
}

Example 7 with NamespaceBundleStats

use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.

the class LoadBalancerTest method newBundleStats.

private NamespaceBundleStats newBundleStats(long topics, int producers, int consumers, double msgRateIn, double msgRateOut, double throughputIn, double throughputOut) {
    NamespaceBundleStats stats = new NamespaceBundleStats();
    stats.topics = topics;
    stats.producerCount = producers;
    stats.consumerCount = consumers;
    stats.msgRateIn = msgRateIn;
    stats.msgRateOut = msgRateOut;
    stats.msgThroughputIn = throughputIn;
    stats.msgThroughputOut = throughputOut;
    return stats;
}

Example 8 with NamespaceBundleStats

use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.

the class SimpleLoadManagerImpl method doLoadShedding.

@Override
public void doLoadShedding() {
    long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
    long comfortLoadLevel = this.getLoadBalancerBrokerComfortLoadThresholdPercentage();
    log.info("Running load shedding task as leader broker, overload threshold {}, comfort loadlevel {}", overloadThreshold, comfortLoadLevel);
    // overloadedRU --> bundleName
    Map<ResourceUnit, String> namespaceBundlesToBeUnloaded = new HashMap<>();
    synchronized (currentLoadReports) {
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit overloadedRU = entry.getKey();
            LoadReport lr = entry.getValue();
            if (isAboveLoadLevel(lr.getSystemResourceUsage(), overloadThreshold)) {
                ResourceType bottleneckResourceType = lr.getBottleneckResourceType();
                Map<String, NamespaceBundleStats> bundleStats = lr.getSortedBundleStats(bottleneckResourceType);
                // 1. owns only one namespace
                if (bundleStats.size() == 1) {
                    // can't unload one namespace, just issue a warning message
                    String bundleName = lr.getBundleStats().keySet().iterator().next();
                    log.warn("HIGH USAGE WARNING : Sole namespace bundle {} is overloading broker {}. " + "No Load Shedding will be done on this broker", bundleName, overloadedRU.getResourceId());
                    continue;
                }
                for (Map.Entry<String, NamespaceBundleStats> bundleStat : bundleStats.entrySet()) {
                    String bundleName = bundleStat.getKey();
                    NamespaceBundleStats stats = bundleStat.getValue();
                    // We need at least one underloaded RU from list of candidates that can host this bundle
                    if (isBrokerAvailableForRebalancing(bundleStat.getKey(), comfortLoadLevel)) {
                        log.info("Namespace bundle {} will be unloaded from overloaded broker {}, bundle stats (topics: {}, producers {}, " + "consumers {}, bandwidthIn {}, bandwidthOut {})", bundleName, overloadedRU.getResourceId(), stats.topics, stats.producerCount, stats.consumerCount, stats.msgThroughputIn, stats.msgThroughputOut);
                        namespaceBundlesToBeUnloaded.put(overloadedRU, bundleName);
                    } else {
                        log.info("Unable to shed load from broker {}, no brokers with enough capacity available " + "for re-balancing {}", overloadedRU.getResourceId(), bundleName);
                    }
                    break;
                }
            }
        }
    }
    unloadNamespacesFromOverLoadedBrokers(namespaceBundlesToBeUnloaded);
}

Example 9 with NamespaceBundleStats

use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method testDoLoadShedding.

@Test(enabled = true)
public void testDoLoadShedding() throws Exception {
    LoadManager loadManager = spy(new SimpleLoadManagerImpl(pulsar1));
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod1-broker7.messaging.gq1.yahoo.com:8080", rd);
    ResourceUnit ru2 = new SimpleResourceUnit("http://prod2-broker7.messaging.gq1.yahoo.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    rus.add(ru2);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    // inject the load report and rankings
    SystemResourceUsage systemResource = new SystemResourceUsage();
    systemResource.setBandwidthIn(new ResourceUsage(90, 100));
    Map<String, NamespaceBundleStats> stats = Maps.newHashMap();
    NamespaceBundleStats nsb1 = new NamespaceBundleStats();
    nsb1.msgRateOut = 10000;
    NamespaceBundleStats nsb2 = new NamespaceBundleStats();
    nsb2.msgRateOut = 10000;
    stats.put("property/cluster/namespace1/0x00000000_0xFFFFFFFF", nsb1);
    stats.put("property/cluster/namespace2/0x00000000_0xFFFFFFFF", nsb2);
    Map<ResourceUnit, com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport1 = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport();
    loadReport1.setSystemResourceUsage(systemResource);
    loadReport1.setBundleStats(stats);
    com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport2 = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport();
    loadReport2.setSystemResourceUsage(new SystemResourceUsage());
    loadReport2.setBundleStats(stats);
    loadReports.put(ru1, loadReport1);
    loadReports.put(ru2, loadReport2);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ((SimpleLoadManagerImpl) loadManager).doLoadShedding();
    verify(loadManager, atLeastOnce()).doLoadShedding();
}

Example 10 with NamespaceBundleStats

use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method testNamespaceBundleStats.

@Test
public void testNamespaceBundleStats() {
    NamespaceBundleStats nsb1 = new NamespaceBundleStats();
    nsb1.msgRateOut = 10000;
    nsb1.producerCount = 1;
    nsb1.consumerCount = 1;
    nsb1.cacheSize = 4;
    nsb1.msgRateIn = 500;
    nsb1.msgThroughputIn = 30;
    nsb1.msgThroughputOut = 30;
    NamespaceBundleStats nsb2 = new NamespaceBundleStats();
    nsb2.msgRateOut = 20000;
    nsb2.producerCount = 300;
    nsb2.consumerCount = 300;
    nsb2.cacheSize = 110000;
    nsb2.msgRateIn = 5000;
    nsb2.msgThroughputIn = 110000.0;
    nsb2.msgThroughputOut = 110000.0;
    assertTrue(nsb1.compareTo(nsb2) == -1);
    assertTrue(nsb1.compareByMsgRate(nsb2) == -1);
    assertTrue(nsb1.compareByTopicConnections(nsb2) == -1);
    assertTrue(nsb1.compareByCacheSize(nsb2) == -1);
    assertTrue(nsb1.compareByBandwidthOut(nsb2) == -1);
    assertTrue(nsb1.compareByBandwidthIn(nsb2) == -1);
}