Examples with NamespaceBundleStats org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats used on opensource projects

Example 11 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class BundleSplitterTask method findBundlesToSplit.

/**
 * Determines which bundles should be split based on various thresholds.
 *
 * @param loadData
 *            Load data to base decisions on (does not have benefit of preallocated data since this may not be the
 *            leader broker).
 * @param localData
 *            Local data for the broker we are splitting on.
 * @param pulsar
 *            Service to use.
 * @return All bundles who have exceeded configured thresholds in number of topics, number of sessions, total
 *         message rates, or total throughput.
 */
@Override
public Set<String> findBundlesToSplit(final LoadData loadData, final PulsarService pulsar) {
    bundleCache.clear();
    final ServiceConfiguration conf = pulsar.getConfiguration();
    int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles();
    long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
    long maxBundleSessions = conf.getLoadBalancerNamespaceBundleMaxSessions();
    long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
    loadData.getBrokerData().forEach((broker, brokerData) -> {
        LocalBrokerData localData = brokerData.getLocalData();
        for (final Map.Entry<String, NamespaceBundleStats> entry : localData.getLastStats().entrySet()) {
            final String bundle = entry.getKey();
            final NamespaceBundleStats stats = entry.getValue();
            double totalMessageRate = 0;
            double totalMessageThroughput = 0;
            // Attempt to consider long-term message data, otherwise effectively ignore.
            if (loadData.getBundleData().containsKey(bundle)) {
                final TimeAverageMessageData longTermData = loadData.getBundleData().get(bundle).getLongTermData();
                totalMessageRate = longTermData.totalMsgRate();
                totalMessageThroughput = longTermData.totalMsgThroughput();
            }
            if (stats.topics > maxBundleTopics || stats.consumerCount + stats.producerCount > maxBundleSessions || totalMessageRate > maxBundleMsgRate || totalMessageThroughput > maxBundleBandwidth) {
                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
                try {
                    final int bundleCount = pulsar.getNamespaceService().getBundleCount(NamespaceName.get(namespace));
                    if (bundleCount < maxBundleCount) {
                        bundleCache.add(bundle);
                    } else {
                        log.warn("Could not split namespace bundle {} because namespace {} has too many bundles: {}", bundle, namespace, bundleCount);
                    }
                } catch (Exception e) {
                    log.warn("Error while getting bundle count for namespace {}", namespace, e);
                }
            }
        }
    });
    return bundleCache;
}

Example 12 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class LoadBalancerTest method writeLoadReportsForDynamicQuota.

private void writeLoadReportsForDynamicQuota(long timestamp) throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        lr.setTimestamp(timestamp);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(5000 * (10 + i * 5), 1024000));
        sru.setBandwidthOut(new ResourceUsage(15000 * (10 + i * 5), 1024000));
        sru.setMemory(new ResourceUsage(25 * (10 + i * 5), 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(200, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            stats.msgRateIn = 5 * (i + j);
            stats.msgRateOut = 15 * (i + j);
            stats.msgThroughputIn = 5000 * (i + j);
            stats.msgThroughputOut = 15000 * (i + j);
            stats.topics = 25 * (i + j);
            stats.consumerCount = 50 * (i + j);
            stats.producerCount = 50 * (i + j);
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
}

Example 13 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class LoadBalancerTest method newBundleStats.

private NamespaceBundleStats newBundleStats(long topics, int producers, int consumers, double msgRateIn, double msgRateOut, double throughputIn, double throughputOut) {
    NamespaceBundleStats stats = new NamespaceBundleStats();
    stats.topics = topics;
    stats.producerCount = producers;
    stats.consumerCount = consumers;
    stats.msgRateIn = msgRateIn;
    stats.msgRateOut = msgRateOut;
    stats.msgThroughputIn = throughputIn;
    stats.msgThroughputOut = throughputOut;
    return stats;
}

Example 14 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class LoadBalancerTest method testTopicAssignmentWithExistingBundles.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - already has some bundles
     * assigned Check the distribution of new topics is roughly consistent (with <10% variation) with the ranking
     */
@Test
public void testTopicAssignmentWithExistingBundles() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        ResourceQuota defaultQuota = new ResourceQuota();
        defaultQuota.setMsgRateIn(20);
        defaultQuota.setMsgRateOut(60);
        defaultQuota.setBandwidthIn(20000);
        defaultQuota.setBandwidthOut(60000);
        defaultQuota.setMemory(87);
        pulsarServices[i].getLocalZkCacheService().getResourceQuotaCache().setDefaultQuota(defaultQuota);
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(0, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < (i + 1) * 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // print ranking
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
    }
    // check owner of new destiations and verify that the distribution is roughly
    // consistent (variation < 10%) with the broker capacity:
    int totalNamespaces = 250;
    int[] expectedAssignments = new int[] { 17, 34, 51, 68, 85 };
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        TopicName topicName = TopicName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().get().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(topicName)).get();
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    double expectedMaxVariation = 10.0;
    for (int i = 0; i < BROKER_COUNT; i++) {
        long actualValue = 0;
        String resourceId = "http://" + lookupAddresses[i];
        if (namespaceOwner.containsKey(resourceId)) {
            actualValue = namespaceOwner.get(resourceId);
        }
        long expectedValue = expectedAssignments[i];
        double variation = Math.abs(actualValue - expectedValue) * 100.0 / expectedValue;
        log.info("Topic assignment - {}, actual: {}, expected baseline: {}, variation: {}/%", lookupAddresses[i], actualValue, expectedValue, String.format("%.2f", variation));
        assertTrue(variation < expectedMaxVariation);
    }
}

Example 15 with NamespaceBundleStats

use of org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testDoLoadShedding.

@Test(enabled = true)
public void testDoLoadShedding() throws Exception {
    LoadManager loadManager = spy(new SimpleLoadManagerImpl(pulsar1));
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://pulsar-broker1.com:8080", rd);
    ResourceUnit ru2 = new SimpleResourceUnit("http://pulsar-broker2.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    rus.add(ru2);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    // inject the load report and rankings
    SystemResourceUsage systemResource = new SystemResourceUsage();
    systemResource.setBandwidthIn(new ResourceUsage(90, 100));
    Map<String, NamespaceBundleStats> stats = Maps.newHashMap();
    NamespaceBundleStats nsb1 = new NamespaceBundleStats();
    nsb1.msgRateOut = 10000;
    NamespaceBundleStats nsb2 = new NamespaceBundleStats();
    nsb2.msgRateOut = 10000;
    stats.put("property/cluster/namespace1/0x00000000_0xFFFFFFFF", nsb1);
    stats.put("property/cluster/namespace2/0x00000000_0xFFFFFFFF", nsb2);
    Map<ResourceUnit, org.apache.pulsar.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport1 = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport1.setSystemResourceUsage(systemResource);
    loadReport1.setBundleStats(stats);
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport2 = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport2.setSystemResourceUsage(new SystemResourceUsage());
    loadReport2.setBundleStats(stats);
    loadReports.put(ru1, loadReport1);
    loadReports.put(ru2, loadReport2);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ((SimpleLoadManagerImpl) loadManager).doLoadShedding();
    verify(loadManager, atLeastOnce()).doLoadShedding();
}