Examples with BrokerData org.apache.pulsar.policies.data.loadbalancer.BrokerData used on opensource projects

Example 1 with BrokerData

use of org.apache.pulsar.policies.data.loadbalancer.BrokerData in project pulsar by yahoo.

the class ModularLoadManagerImplTest method testLoadShedding.

// Test that load shedding works
@Test
public void testLoadShedding() throws Exception {
    final NamespaceBundleStats stats1 = new NamespaceBundleStats();
    final NamespaceBundleStats stats2 = new NamespaceBundleStats();
    stats1.msgRateIn = 100;
    stats2.msgRateIn = 200;
    final Map<String, NamespaceBundleStats> statsMap = new ConcurrentHashMap<>();
    statsMap.put(mockBundleName(1), stats1);
    statsMap.put(mockBundleName(2), stats2);
    final LocalBrokerData localBrokerData = new LocalBrokerData();
    localBrokerData.update(new SystemResourceUsage(), statsMap);
    final Namespaces namespacesSpy1 = spy(pulsar1.getAdminClient().namespaces());
    AtomicReference<String> bundleReference = new AtomicReference<>();
    doAnswer(invocation -> {
        bundleReference.set(invocation.getArguments()[0].toString() + '/' + invocation.getArguments()[1]);
        return null;
    }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1);
    pulsar1.getConfiguration().setLoadBalancerEnabled(true);
    final LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData");
    final Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost));
    when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData);
    brokerDataMap.put(primaryHost, brokerDataSpy1);
    // Need to update all the bundle data for the shredder to see the spy.
    primaryLoadManager.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080"));
    Thread.sleep(100);
    localBrokerData.setCpu(new ResourceUsage(80, 100));
    primaryLoadManager.doLoadShedding();
    // 80% is below overload threshold: verify nothing is unloaded.
    verify(namespacesSpy1, Mockito.times(0)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    localBrokerData.setCpu(new ResourceUsage(90, 100));
    primaryLoadManager.doLoadShedding();
    // Most expensive bundle will be unloaded.
    verify(namespacesSpy1, Mockito.times(1)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    assertEquals(bundleReference.get(), mockBundleName(2));
    primaryLoadManager.doLoadShedding();
    // Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be
    // unloaded, but this is not the case due to the spy's behavior).
    verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    assertEquals(bundleReference.get(), mockBundleName(1));
    primaryLoadManager.doLoadShedding();
    // Now both are in grace period: neither should be unloaded.
    verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
}

Example 2 with BrokerData

use of org.apache.pulsar.policies.data.loadbalancer.BrokerData in project pulsar by yahoo.

the class ModularLoadManagerStrategyTest method testLeastLongTermMessageRate.

// Test that least long term message rate works correctly.
public void testLeastLongTermMessageRate() {
    BundleData bundleData = new BundleData();
    BrokerData brokerData1 = initBrokerData();
    BrokerData brokerData2 = initBrokerData();
    BrokerData brokerData3 = initBrokerData();
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(100);
    brokerData2.getTimeAverageData().setLongTermMsgRateIn(200);
    brokerData3.getTimeAverageData().setLongTermMsgRateIn(300);
    LoadData loadData = new LoadData();
    Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    brokerDataMap.put("1", brokerData1);
    brokerDataMap.put("2", brokerData2);
    brokerDataMap.put("3", brokerData3);
    ServiceConfiguration conf = new ServiceConfiguration();
    ModularLoadManagerStrategy strategy = new LeastLongTermMessageRate(conf);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("1"));
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(400);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2"));
    brokerData2.getLocalData().setCpu(new ResourceUsage(90, 100));
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("3"));
}

Example 3 with BrokerData

use of org.apache.pulsar.policies.data.loadbalancer.BrokerData in project pulsar by yahoo.

the class ModularLoadManagerStrategyTest method initBrokerData.

private BrokerData initBrokerData() {
    LocalBrokerData localBrokerData = new LocalBrokerData();
    localBrokerData.setCpu(new ResourceUsage());
    localBrokerData.setMemory(new ResourceUsage());
    localBrokerData.setBandwidthIn(new ResourceUsage());
    localBrokerData.setBandwidthOut(new ResourceUsage());
    BrokerData brokerData = new BrokerData(localBrokerData);
    TimeAverageBrokerData timeAverageBrokerData = new TimeAverageBrokerData();
    brokerData.setTimeAverageData(timeAverageBrokerData);
    return brokerData;
}

Example 4 with BrokerData

use of org.apache.pulsar.policies.data.loadbalancer.BrokerData in project pulsar by yahoo.

the class OverloadShedderTest method testBrokerWithMultipleBundles.

@Test
public void testBrokerWithMultipleBundles() {
    int numBundles = 10;
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(999, 1000));
    broker1.setBandwidthOut(new ResourceUsage(999, 1000));
    LocalBrokerData anotherBroker = new LocalBrokerData();
    String anotherBrokerName = "another-broker";
    double brokerThroghput = 0;
    for (int i = 1; i <= numBundles; i++) {
        broker1.getBundles().add("bundle-" + i);
        BundleData bundle = new BundleData();
        TimeAverageMessageData db = new TimeAverageMessageData();
        double throughput = i * 1024 * 1024;
        db.setMsgThroughputIn(throughput);
        db.setMsgThroughputOut(throughput);
        bundle.setShortTermData(db);
        loadData.getBundleData().put("bundle-" + i, bundle);
        // This bundle should not be selected for `broker1` since it is belong to another broker.
        String anotherBundleName = anotherBrokerName + "-bundle-" + (numBundles + i);
        loadData.getBundleData().put(anotherBundleName, bundle);
        anotherBroker.getBundles().add(anotherBundleName);
        brokerThroghput += throughput;
    }
    broker1.setMsgThroughputIn(brokerThroghput);
    broker1.setMsgThroughputOut(brokerThroghput);
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    loadData.getBrokerData().put(anotherBrokerName, new BrokerData(anotherBroker));
    Multimap<String, String> bundlesToUnload = os.findBundlesForUnloading(loadData, conf);
    assertFalse(bundlesToUnload.isEmpty());
    assertEquals(bundlesToUnload.get("broker-1"), Lists.newArrayList("bundle-10", "bundle-9"));
}

Example 5 with BrokerData

use of org.apache.pulsar.policies.data.loadbalancer.BrokerData in project pulsar by yahoo.

the class OverloadShedderTest method testBrokersWithNoBundles.

@Test
public void testBrokersWithNoBundles() {
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(999, 1000));
    broker1.setBandwidthOut(new ResourceUsage(999, 1000));
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    assertTrue(os.findBundlesForUnloading(loadData, conf).isEmpty());
}