Examples with BundleData org.apache.pulsar.broker.BundleData used on opensource projects

Example 6 with BundleData

use of org.apache.pulsar.broker.BundleData in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testEvenBundleDistribution.

// Test that bundles belonging to the same namespace are distributed evenly among brokers.
// Test disabled since it's depending on CPU usage in the machine
@Test(enabled = false)
public void testEvenBundleDistribution() throws Exception {
    final NamespaceBundle[] bundles = LoadBalancerTestingUtils.makeBundles(nsFactory, "test", "test", "test", 16);
    int numAssignedToPrimary = 0;
    int numAssignedToSecondary = 0;
    final BundleData bundleData = new BundleData(10, 1000);
    final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000);
    longTermMessageData.setMsgRateIn(1000);
    bundleData.setLongTermData(longTermMessageData);
    final String firstBundleDataPath = String.format("%s/%s", ModularLoadManagerImpl.BUNDLE_DATA_ZPATH, bundles[0]);
    // Write long message rate for first bundle to ensure that even bundle distribution is not a coincidence of
    // balancing by message rate. If we were balancing by message rate, one of the brokers should only have this
    // one bundle.
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), firstBundleDataPath, bundleData.getJsonBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    for (final NamespaceBundle bundle : bundles) {
        if (primaryLoadManager.selectBrokerForAssignment(bundle).equals(primaryHost)) {
            ++numAssignedToPrimary;
        } else {
            ++numAssignedToSecondary;
        }
        if ((numAssignedToPrimary + numAssignedToSecondary) % 2 == 0) {
            // On even number of assignments, assert that an equal number of bundles have been assigned between
            // them.
            assertEquals(numAssignedToPrimary, numAssignedToSecondary);
        }
    }
}

Example 7 with BundleData

use of org.apache.pulsar.broker.BundleData in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testMaxTopicDistributionToBroker.

/**
 * It verifies that once broker owns max-number of topics: load-manager doesn't allocates new bundles to that broker
 * unless all the brokers are in same state.
 *
 * <pre>
 * 1. Create a bundle whose bundle-resource-quota will contain max-topics
 * 2. Load-manager assigns broker to this bundle so, assigned broker is overloaded with max-topics
 * 3. For any new further bundles: broker assigns different brokers.
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testMaxTopicDistributionToBroker() throws Exception {
    final int totalBundles = 50;
    final NamespaceBundle[] bundles = LoadBalancerTestingUtils.makeBundles(nsFactory, "test", "test", "test", totalBundles);
    final BundleData bundleData = new BundleData(10, 1000);
    // it sets max topics under this bundle so, owner of this broker reaches max-topic threshold
    bundleData.setTopics(pulsar1.getConfiguration().getLoadBalancerBrokerMaxTopics() + 10);
    final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000);
    longTermMessageData.setMsgRateIn(1000);
    bundleData.setLongTermData(longTermMessageData);
    final String firstBundleDataPath = String.format("%s/%s", ModularLoadManagerImpl.BUNDLE_DATA_ZPATH, bundles[0]);
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), firstBundleDataPath, bundleData.getJsonBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    String maxTopicOwnedBroker = primaryLoadManager.selectBrokerForAssignment(bundles[0]).get();
    for (int i = 1; i < totalBundles; i++) {
        assertNotEquals(primaryLoadManager.selectBrokerForAssignment(bundles[i]), maxTopicOwnedBroker);
    }
}

Example 8 with BundleData

use of org.apache.pulsar.broker.BundleData in project incubator-pulsar by apache.

the class ModularLoadManagerStrategyTest method testLeastLongTermMessageRate.

// Test that least long term message rate works correctly.
@Test
public void testLeastLongTermMessageRate() {
    BundleData bundleData = new BundleData();
    BrokerData brokerData1 = initBrokerData();
    BrokerData brokerData2 = initBrokerData();
    BrokerData brokerData3 = initBrokerData();
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(100);
    brokerData2.getTimeAverageData().setLongTermMsgRateIn(200);
    brokerData3.getTimeAverageData().setLongTermMsgRateIn(300);
    LoadData loadData = new LoadData();
    Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    brokerDataMap.put("1", brokerData1);
    brokerDataMap.put("2", brokerData2);
    brokerDataMap.put("3", brokerData3);
    ServiceConfiguration conf = new ServiceConfiguration();
    ModularLoadManagerStrategy strategy = new LeastLongTermMessageRate(conf);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("1"));
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(400);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2"));
    brokerData2.getLocalData().setCpu(new ResourceUsage(90, 100));
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("3"));
}

Example 9 with BundleData

use of org.apache.pulsar.broker.BundleData in project incubator-pulsar by apache.

the class LoadSimulationController method handleSimulate.

// Handle the command line arguments associated with the simulate command.
private void handleSimulate(final ShellArguments arguments) throws Exception {
    final List<String> commandArguments = arguments.commandArguments;
    checkAppArgs(commandArguments.size() - 1, 1);
    final ZooKeeper zkClient = new ZooKeeper(commandArguments.get(1), 5000, null);
    // Make a map for each thread to speed up the ZooKeeper writing process.
    final Map<String, ResourceQuota>[] threadLocalMaps = new Map[clients.length];
    for (int i = 0; i < clients.length; ++i) {
        threadLocalMaps[i] = new HashMap<>();
    }
    getResourceQuotas(QUOTA_ROOT, zkClient, threadLocalMaps);
    final List<Future> futures = new ArrayList<>(clients.length);
    int i = 0;
    log.info("Simulating...");
    for (final Map<String, ResourceQuota> bundleToQuota : threadLocalMaps) {
        final int j = i;
        futures.add(threadPool.submit(() -> {
            for (final Map.Entry<String, ResourceQuota> entry : bundleToQuota.entrySet()) {
                final String bundle = entry.getKey();
                final String newAPIPath = bundle.replace(QUOTA_ROOT, BUNDLE_DATA_ROOT);
                final ResourceQuota quota = entry.getValue();
                final int tenantStart = QUOTA_ROOT.length() + 1;
                final String topic = String.format("persistent://%s/t", bundle.substring(tenantStart));
                final BundleData bundleData = initializeBundleData(quota, arguments);
                // Put the bundle data in the new ZooKeeper.
                try {
                    ZkUtils.createFullPathOptimistic(zkClient, newAPIPath, bundleData.getJsonBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
                } catch (KeeperException.NodeExistsException e) {
                    try {
                        zkClient.setData(newAPIPath, bundleData.getJsonBytes(), -1);
                    } catch (Exception ex) {
                        throw new RuntimeException(ex);
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
                try {
                    trade(arguments, topic, j);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        }));
        ++i;
    }
    for (final Future future : futures) {
        future.get();
    }
    zkClient.close();
}

Example 10 with BundleData

use of org.apache.pulsar.broker.BundleData in project incubator-pulsar by apache.

the class LoadSimulationController method initializeBundleData.

// Initialize a BundleData from a resource quota and configurations and modify the quota accordingly.
private BundleData initializeBundleData(final ResourceQuota quota, final ShellArguments arguments) {
    final double messageRate = (quota.getMsgRateIn() + quota.getMsgRateOut()) / 2;
    final int messageSize = (int) Math.ceil((quota.getBandwidthIn() + quota.getBandwidthOut()) / (2 * messageRate));
    arguments.rate = messageRate * arguments.rateMultiplier;
    arguments.size = messageSize;
    final NamespaceBundleStats startingStats = new NamespaceBundleStats();
    // Modify the original quota so that new rates are set.
    final double modifiedRate = messageRate * arguments.rateMultiplier;
    final double modifiedBandwidth = (quota.getBandwidthIn() + quota.getBandwidthOut()) * arguments.rateMultiplier / 2;
    quota.setMsgRateIn(modifiedRate);
    quota.setMsgRateOut(modifiedRate);
    quota.setBandwidthIn(modifiedBandwidth);
    quota.setBandwidthOut(modifiedBandwidth);
    // Assume modified memory usage is comparable to the rate multiplier times the original usage.
    quota.setMemory(quota.getMemory() * arguments.rateMultiplier);
    startingStats.msgRateIn = quota.getMsgRateIn();
    startingStats.msgRateOut = quota.getMsgRateOut();
    startingStats.msgThroughputIn = quota.getBandwidthIn();
    startingStats.msgThroughputOut = quota.getBandwidthOut();
    final BundleData bundleData = new BundleData(10, 1000, startingStats);
    // Assume there is ample history for the bundle.
    bundleData.getLongTermData().setNumSamples(1000);
    bundleData.getShortTermData().setNumSamples(10);
    return bundleData;
}