Examples with BundleData org.apache.pulsar.policies.data.loadbalancer.BundleData used on opensource projects

Example 1 with BundleData

use of org.apache.pulsar.policies.data.loadbalancer.BundleData in project pulsar by yahoo.

the class LoadSimulationController method initializeBundleData.

// Initialize a BundleData from a resource quota and configurations and modify the quota accordingly.
private BundleData initializeBundleData(final ResourceQuota quota, final ShellArguments arguments) {
    final double messageRate = (quota.getMsgRateIn() + quota.getMsgRateOut()) / 2;
    final int messageSize = (int) Math.ceil((quota.getBandwidthIn() + quota.getBandwidthOut()) / (2 * messageRate));
    arguments.rate = messageRate * arguments.rateMultiplier;
    arguments.size = messageSize;
    final NamespaceBundleStats startingStats = new NamespaceBundleStats();
    // Modify the original quota so that new rates are set.
    final double modifiedRate = messageRate * arguments.rateMultiplier;
    final double modifiedBandwidth = (quota.getBandwidthIn() + quota.getBandwidthOut()) * arguments.rateMultiplier / 2;
    quota.setMsgRateIn(modifiedRate);
    quota.setMsgRateOut(modifiedRate);
    quota.setBandwidthIn(modifiedBandwidth);
    quota.setBandwidthOut(modifiedBandwidth);
    // Assume modified memory usage is comparable to the rate multiplier times the original usage.
    quota.setMemory(quota.getMemory() * arguments.rateMultiplier);
    startingStats.msgRateIn = quota.getMsgRateIn();
    startingStats.msgRateOut = quota.getMsgRateOut();
    startingStats.msgThroughputIn = quota.getBandwidthIn();
    startingStats.msgThroughputOut = quota.getBandwidthOut();
    final BundleData bundleData = new BundleData(10, 1000, startingStats);
    // Assume there is ample history for the bundle.
    bundleData.getLongTermData().setNumSamples(1000);
    bundleData.getShortTermData().setNumSamples(10);
    return bundleData;
}

Example 2 with BundleData

use of org.apache.pulsar.policies.data.loadbalancer.BundleData in project pulsar by yahoo.

the class ModularLoadManagerStrategyTest method testLeastLongTermMessageRate.

// Test that least long term message rate works correctly.
public void testLeastLongTermMessageRate() {
    BundleData bundleData = new BundleData();
    BrokerData brokerData1 = initBrokerData();
    BrokerData brokerData2 = initBrokerData();
    BrokerData brokerData3 = initBrokerData();
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(100);
    brokerData2.getTimeAverageData().setLongTermMsgRateIn(200);
    brokerData3.getTimeAverageData().setLongTermMsgRateIn(300);
    LoadData loadData = new LoadData();
    Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    brokerDataMap.put("1", brokerData1);
    brokerDataMap.put("2", brokerData2);
    brokerDataMap.put("3", brokerData3);
    ServiceConfiguration conf = new ServiceConfiguration();
    ModularLoadManagerStrategy strategy = new LeastLongTermMessageRate(conf);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("1"));
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(400);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2"));
    brokerData2.getLocalData().setCpu(new ResourceUsage(90, 100));
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("3"));
}

Example 3 with BundleData

use of org.apache.pulsar.policies.data.loadbalancer.BundleData in project pulsar by yahoo.

the class OverloadShedderTest method testBrokerWithMultipleBundles.

@Test
public void testBrokerWithMultipleBundles() {
    int numBundles = 10;
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(999, 1000));
    broker1.setBandwidthOut(new ResourceUsage(999, 1000));
    LocalBrokerData anotherBroker = new LocalBrokerData();
    String anotherBrokerName = "another-broker";
    double brokerThroghput = 0;
    for (int i = 1; i <= numBundles; i++) {
        broker1.getBundles().add("bundle-" + i);
        BundleData bundle = new BundleData();
        TimeAverageMessageData db = new TimeAverageMessageData();
        double throughput = i * 1024 * 1024;
        db.setMsgThroughputIn(throughput);
        db.setMsgThroughputOut(throughput);
        bundle.setShortTermData(db);
        loadData.getBundleData().put("bundle-" + i, bundle);
        // This bundle should not be selected for `broker1` since it is belong to another broker.
        String anotherBundleName = anotherBrokerName + "-bundle-" + (numBundles + i);
        loadData.getBundleData().put(anotherBundleName, bundle);
        anotherBroker.getBundles().add(anotherBundleName);
        brokerThroghput += throughput;
    }
    broker1.setMsgThroughputIn(brokerThroghput);
    broker1.setMsgThroughputOut(brokerThroghput);
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    loadData.getBrokerData().put(anotherBrokerName, new BrokerData(anotherBroker));
    Multimap<String, String> bundlesToUnload = os.findBundlesForUnloading(loadData, conf);
    assertFalse(bundlesToUnload.isEmpty());
    assertEquals(bundlesToUnload.get("broker-1"), Lists.newArrayList("bundle-10", "bundle-9"));
}

Example 4 with BundleData

use of org.apache.pulsar.policies.data.loadbalancer.BundleData in project pulsar by yahoo.

the class OverloadShedderTest method testBrokerWithSingleBundle.

@Test
public void testBrokerWithSingleBundle() {
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(999, 1000));
    broker1.setBandwidthOut(new ResourceUsage(999, 1000));
    broker1.setBundles(Sets.newHashSet("bundle-1"));
    BundleData bundle1 = new BundleData();
    TimeAverageMessageData db1 = new TimeAverageMessageData();
    db1.setMsgThroughputIn(1000);
    db1.setMsgThroughputOut(1000);
    bundle1.setShortTermData(db1);
    loadData.getBundleData().put("bundle-1", bundle1);
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    assertTrue(os.findBundlesForUnloading(loadData, conf).isEmpty());
}

Example 5 with BundleData

use of org.apache.pulsar.policies.data.loadbalancer.BundleData in project pulsar by yahoo.

the class OverloadShedderTest method testFilterRecentlyUnloaded.

@Test
public void testFilterRecentlyUnloaded() {
    int numBundles = 10;
    LoadData loadData = new LoadData();
    LocalBrokerData broker1 = new LocalBrokerData();
    broker1.setBandwidthIn(new ResourceUsage(999, 1000));
    broker1.setBandwidthOut(new ResourceUsage(999, 1000));
    double brokerThroghput = 0;
    for (int i = 1; i <= numBundles; i++) {
        broker1.getBundles().add("bundle-" + i);
        BundleData bundle = new BundleData();
        TimeAverageMessageData db = new TimeAverageMessageData();
        double throughput = i * 1024 * 1024;
        db.setMsgThroughputIn(throughput);
        db.setMsgThroughputOut(throughput);
        bundle.setShortTermData(db);
        loadData.getBundleData().put("bundle-" + i, bundle);
        brokerThroghput += throughput;
    }
    broker1.setMsgThroughputIn(brokerThroghput);
    broker1.setMsgThroughputOut(brokerThroghput);
    loadData.getBrokerData().put("broker-1", new BrokerData(broker1));
    loadData.getRecentlyUnloadedBundles().put("bundle-10", 1L);
    loadData.getRecentlyUnloadedBundles().put("bundle-9", 1L);
    Multimap<String, String> bundlesToUnload = os.findBundlesForUnloading(loadData, conf);
    assertFalse(bundlesToUnload.isEmpty());
    assertEquals(bundlesToUnload.get("broker-1"), Lists.newArrayList("bundle-8", "bundle-7"));
}