Examples with LocalBrokerData org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData used on opensource projects

Example 11 with LocalBrokerData

use of org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method initialize.

/**
 * Initialize this load manager using the given PulsarService. Should be called only once, after invoking the
 * default constructor.
 *
 * @param pulsar
 *            The service to initialize with.
 */
public void initialize(final PulsarService pulsar) {
    this.pulsar = pulsar;
    availableActiveBrokers = new ZooKeeperChildrenCache(pulsar.getLocalZkCache(), LoadManager.LOADBALANCE_BROKERS_ROOT);
    availableActiveBrokers.registerListener(new ZooKeeperCacheListener<Set<String>>() {

        @Override
        public void onUpdate(String path, Set<String> data, Stat stat) {
            if (log.isDebugEnabled()) {
                log.debug("Update Received for path {}", path);
            }
            reapDeadBrokerPreallocations(data);
            scheduler.submit(ModularLoadManagerImpl.this::updateAll);
        }
    });
    brokerDataCache = new ZooKeeperDataCache<LocalBrokerData>(pulsar.getLocalZkCache()) {

        @Override
        public LocalBrokerData deserialize(String key, byte[] content) throws Exception {
            return ObjectMapperFactory.getThreadLocal().readValue(content, LocalBrokerData.class);
        }
    };
    brokerDataCache.registerListener(this);
    if (SystemUtils.IS_OS_LINUX) {
        brokerHostUsage = new LinuxBrokerHostUsageImpl(pulsar);
    } else {
        brokerHostUsage = new GenericBrokerHostUsageImpl(pulsar);
    }
    bundleSplitStrategy = new BundleSplitterTask(pulsar);
    conf = pulsar.getConfiguration();
    // Initialize the default stats to assume for unseen bundles (hard-coded for now).
    defaultStats.msgThroughputIn = DEFAULT_MESSAGE_THROUGHPUT;
    defaultStats.msgThroughputOut = DEFAULT_MESSAGE_THROUGHPUT;
    defaultStats.msgRateIn = DEFAULT_MESSAGE_RATE;
    defaultStats.msgRateOut = DEFAULT_MESSAGE_RATE;
    lastData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
    localData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
    localData.setBrokerVersionString(pulsar.getBrokerVersion());
    // configure broker-topic mode
    lastData.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
    lastData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
    localData.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
    localData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
    placementStrategy = ModularLoadManagerStrategy.create(conf);
    policies = new SimpleResourceAllocationPolicies(pulsar);
    zkClient = pulsar.getZkClient();
    filterPipeline.add(new BrokerVersionFilter());
    refreshBrokerToFailureDomainMap();
    // register listeners for domain changes
    pulsar.getConfigurationCache().failureDomainListCache().registerListener((path, data, stat) -> scheduler.execute(() -> refreshBrokerToFailureDomainMap()));
    pulsar.getConfigurationCache().failureDomainCache().registerListener((path, data, stat) -> scheduler.execute(() -> refreshBrokerToFailureDomainMap()));
}

Example 12 with LocalBrokerData

use of org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData in project incubator-pulsar by apache.

the class BundleSplitterTask method findBundlesToSplit.

/**
 * Determines which bundles should be split based on various thresholds.
 *
 * @param loadData
 *            Load data to base decisions on (does not have benefit of preallocated data since this may not be the
 *            leader broker).
 * @param localData
 *            Local data for the broker we are splitting on.
 * @param pulsar
 *            Service to use.
 * @return All bundles who have exceeded configured thresholds in number of topics, number of sessions, total
 *         message rates, or total throughput.
 */
@Override
public Set<String> findBundlesToSplit(final LoadData loadData, final PulsarService pulsar) {
    bundleCache.clear();
    final ServiceConfiguration conf = pulsar.getConfiguration();
    int maxBundleCount = conf.getLoadBalancerNamespaceMaximumBundles();
    long maxBundleTopics = conf.getLoadBalancerNamespaceBundleMaxTopics();
    long maxBundleSessions = conf.getLoadBalancerNamespaceBundleMaxSessions();
    long maxBundleMsgRate = conf.getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBundleBandwidth = conf.getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * LoadManagerShared.MIBI;
    loadData.getBrokerData().forEach((broker, brokerData) -> {
        LocalBrokerData localData = brokerData.getLocalData();
        for (final Map.Entry<String, NamespaceBundleStats> entry : localData.getLastStats().entrySet()) {
            final String bundle = entry.getKey();
            final NamespaceBundleStats stats = entry.getValue();
            double totalMessageRate = 0;
            double totalMessageThroughput = 0;
            // Attempt to consider long-term message data, otherwise effectively ignore.
            if (loadData.getBundleData().containsKey(bundle)) {
                final TimeAverageMessageData longTermData = loadData.getBundleData().get(bundle).getLongTermData();
                totalMessageRate = longTermData.totalMsgRate();
                totalMessageThroughput = longTermData.totalMsgThroughput();
            }
            if (stats.topics > maxBundleTopics || stats.consumerCount + stats.producerCount > maxBundleSessions || totalMessageRate > maxBundleMsgRate || totalMessageThroughput > maxBundleBandwidth) {
                final String namespace = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
                try {
                    final int bundleCount = pulsar.getNamespaceService().getBundleCount(NamespaceName.get(namespace));
                    if (bundleCount < maxBundleCount) {
                        bundleCache.add(bundle);
                    } else {
                        log.warn("Could not split namespace bundle {} because namespace {} has too many bundles: {}", bundle, namespace, bundleCount);
                    }
                } catch (Exception e) {
                    log.warn("Error while getting bundle count for namespace {}", namespace, e);
                }
            }
        }
    });
    return bundleCache;
}

Example 13 with LocalBrokerData

use of org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData in project incubator-pulsar by apache.

the class ModularLoadManagerStrategyTest method initBrokerData.

private BrokerData initBrokerData() {
    LocalBrokerData localBrokerData = new LocalBrokerData();
    localBrokerData.setCpu(new ResourceUsage());
    localBrokerData.setMemory(new ResourceUsage());
    localBrokerData.setBandwidthIn(new ResourceUsage());
    localBrokerData.setBandwidthOut(new ResourceUsage());
    BrokerData brokerData = new BrokerData(localBrokerData);
    TimeAverageBrokerData timeAverageBrokerData = new TimeAverageBrokerData();
    brokerData.setTimeAverageData(timeAverageBrokerData);
    return brokerData;
}

Example 14 with LocalBrokerData

use of org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData in project incubator-pulsar by apache.

the class NamespaceServiceTest method testLoadReportDeserialize.

/**
 * <pre>
 *  It verifies that namespace service deserialize the load-report based on load-manager which active.
 *  1. write candidate1- load report using {@link LoadReport} which is used by SimpleLoadManagerImpl
 *  2. Write candidate2- load report using {@link LocalBrokerData} which is used by ModularLoadManagerImpl
 *  3. try to get Lookup Result based on active load-manager
 * </pre>
 * @throws Exception
 */
@Test
public void testLoadReportDeserialize() throws Exception {
    final String candidateBroker1 = "http://localhost:8000";
    final String candidateBroker2 = "http://localhost:3000";
    LoadReport lr = new LoadReport(null, null, candidateBroker1, null);
    LocalBrokerData ld = new LocalBrokerData(null, null, candidateBroker2, null);
    URI uri1 = new URI(candidateBroker1);
    URI uri2 = new URI(candidateBroker2);
    String path1 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri1.getHost(), uri1.getPort());
    String path2 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri2.getHost(), uri2.getPort());
    ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), path1, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(lr), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
    ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), path2, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(ld), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
    LookupResult result1 = pulsar.getNamespaceService().createLookupResult(candidateBroker1).get();
    // update to new load mananger
    pulsar.getLoadManager().set(new ModularLoadManagerWrapper(new ModularLoadManagerImpl()));
    LookupResult result2 = pulsar.getNamespaceService().createLookupResult(candidateBroker2).get();
    Assert.assertEquals(result1.getLookupData().getBrokerUrl(), candidateBroker1);
    Assert.assertEquals(result2.getLookupData().getBrokerUrl(), candidateBroker2);
    System.out.println(result2);
}