Examples with LoadReport org.apache.pulsar.policies.data.loadbalancer.LoadReport used on opensource projects

Example 16 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class ZookeeperCacheLoaderTest method testZookeeperCacheLoader.

/**
 * Create znode for available broker in ZooKeeper and updates it again to verify ZooKeeper cache update
 *
 * @throws InterruptedException
 * @throws KeeperException
 * @throws IOException
 */
@Test
public void testZookeeperCacheLoader() throws InterruptedException, KeeperException, Exception {
    DiscoveryZooKeeperClientFactoryImpl.zk = mockZookKeeper;
    @SuppressWarnings("resource") ZookeeperCacheLoader zkLoader = new ZookeeperCacheLoader(new DiscoveryZooKeeperClientFactoryImpl(), "", 30_000);
    List<String> brokers = Lists.newArrayList("broker-1:15000", "broker-2:15000", "broker-3:15000");
    for (int i = 0; i < brokers.size(); i++) {
        try {
            LoadManagerReport report = i % 2 == 0 ? getSimpleLoadManagerLoadReport(brokers.get(i)) : getModularLoadManagerLoadReport(brokers.get(i));
            zkLoader.getLocalZkCache().getZooKeeper().create(LOADBALANCE_BROKERS_ROOT + "/" + brokers.get(i), ObjectMapperFactory.getThreadLocal().writeValueAsBytes(report), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
        } catch (Exception e) {
            fail("failed while creating broker znodes");
        }
    }
    // strategically wait for cache to get sync
    for (int i = 0; i < 5; i++) {
        if (zkLoader.getAvailableBrokers().size() == 3 || i == 4) {
            break;
        }
        Thread.sleep(1000);
    }
    // 2. get available brokers from ZookeeperCacheLoader
    List<LoadManagerReport> list = zkLoader.getAvailableBrokers();
    // 3. verify retrieved broker list
    Set<String> cachedBrokers = list.stream().map(loadReport -> loadReport.getWebServiceUrl()).collect(Collectors.toSet());
    Assert.assertEquals(list.size(), brokers.size());
    Assert.assertTrue(brokers.containsAll(cachedBrokers));
    // 4.a add new broker
    final String newBroker = "broker-4:15000";
    LoadManagerReport report = getSimpleLoadManagerLoadReport(newBroker);
    zkLoader.getLocalZkCache().getZooKeeper().create(LOADBALANCE_BROKERS_ROOT + "/" + newBroker, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(report), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    brokers.add(newBroker);
    // wait for 100 msec: to get cache updated
    Thread.sleep(100);
    // 4.b. get available brokers from ZookeeperCacheLoader
    list = zkLoader.getAvailableBrokers();
    // 4.c. verify retrieved broker list
    cachedBrokers = list.stream().map(loadReport -> loadReport.getWebServiceUrl()).collect(Collectors.toSet());
    Assert.assertEquals(list.size(), brokers.size());
    Assert.assertTrue(brokers.containsAll(cachedBrokers));
}

Example 17 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method initialize.

@Override
public void initialize(final PulsarService pulsar) {
    if (SystemUtils.IS_OS_LINUX) {
        brokerHostUsage = new LinuxBrokerHostUsageImpl(pulsar);
    } else {
        brokerHostUsage = new GenericBrokerHostUsageImpl(pulsar);
    }
    this.policies = new SimpleResourceAllocationPolicies(pulsar);
    lastLoadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
    lastLoadReport.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
    lastLoadReport.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
    loadReportCacheZk = new ZooKeeperDataCache<LoadReport>(pulsar.getLocalZkCache()) {

        @Override
        public LoadReport deserialize(String key, byte[] content) throws Exception {
            return ObjectMapperFactory.getThreadLocal().readValue(content, LoadReport.class);
        }
    };
    loadReportCacheZk.registerListener(this);
    this.dynamicConfigurationCache = new ZooKeeperDataCache<Map<String, String>>(pulsar.getLocalZkCache()) {

        @Override
        public Map<String, String> deserialize(String key, byte[] content) throws Exception {
            return ObjectMapperFactory.getThreadLocal().readValue(content, HashMap.class);
        }
    };
    int entryExpiryTime = (int) pulsar.getConfiguration().getLoadBalancerSheddingGracePeriodMinutes();
    unloadedHotNamespaceCache = CacheBuilder.newBuilder().expireAfterWrite(entryExpiryTime, TimeUnit.MINUTES).build(new CacheLoader<String, Long>() {

        @Override
        public Long load(String key) throws Exception {
            return System.currentTimeMillis();
        }
    });
    availableActiveBrokers = new ZooKeeperChildrenCache(pulsar.getLocalZkCache(), LOADBALANCE_BROKERS_ROOT);
    availableActiveBrokers.registerListener(new ZooKeeperCacheListener<Set<String>>() {

        @Override
        public void onUpdate(String path, Set<String> data, Stat stat) {
            if (log.isDebugEnabled()) {
                log.debug("Update Received for path {}", path);
            }
            scheduler.submit(SimpleLoadManagerImpl.this::updateRanking);
        }
    });
    this.pulsar = pulsar;
}

Example 18 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method doLoadRanking.

/**
 * Rank brokers by available capacity, or load percentage, based on placement strategy:
 *
 * - Available capacity for weighted random selection (weightedRandomSelection): ranks ResourceUnits units based on
 * estimation of their capacity which is basically how many bundles each ResourceUnit is able can handle with its
 * available resources (CPU, memory, network, etc);
 *
 * - Load percentage for least loaded server (leastLoadedServer): ranks ResourceUnits units based on estimation of
 * their load percentage which is basically how many percent of resource is allocated which is
 * max(resource_actually_used, resource_quota)
 *
 * If we fail to collect the Load Reports OR fail to process them for the first time, it means the leader does not
 * have enough information to make a decision so we set it to ready when we collect and process the load reports
 * successfully the first time.
 */
private synchronized void doLoadRanking() {
    ResourceUnitRanking.setCpuUsageByMsgRate(this.realtimeCpuLoadFactor);
    String hostname = pulsar.getAdvertisedAddress();
    String strategy = this.getLoadBalancerPlacementStrategy();
    log.info("doLoadRanking - load balancing strategy: {}", strategy);
    if (!currentLoadReports.isEmpty()) {
        Map<Long, Set<ResourceUnit>> newSortedRankings = Maps.newTreeMap();
        Map<ResourceUnit, ResourceUnitRanking> newResourceUnitRankings = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            // calculate rankings
            Set<String> loadedBundles = loadReport.getBundles();
            Set<String> preAllocatedBundles = null;
            if (resourceUnitRankings.containsKey(resourceUnit)) {
                preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles();
                preAllocatedBundles.removeAll(loadedBundles);
            } else {
                preAllocatedBundles = new HashSet<>();
            }
            ResourceQuota allocatedQuota = getTotalAllocatedQuota(loadedBundles);
            ResourceQuota preAllocatedQuota = getTotalAllocatedQuota(preAllocatedBundles);
            ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), loadedBundles, allocatedQuota, preAllocatedBundles, preAllocatedQuota);
            newResourceUnitRankings.put(resourceUnit, ranking);
            // generated sorted ranking
            double loadPercentage = ranking.getEstimatedLoadPercentage();
            long maxCapacity = ranking.estimateMaxCapacity(pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota());
            long finalRank = 0;
            if (strategy.equals(LOADBALANCER_STRATEGY_LLS)) {
                finalRank = (long) loadPercentage;
            } else if (strategy.equals(LOADBALANCER_STRATEGY_LEAST_MSG)) {
                finalRank = (long) ranking.getEstimatedMessageRate();
            } else {
                double idleRatio = (100 - loadPercentage) / 100;
                finalRank = (long) (maxCapacity * idleRatio * idleRatio);
            }
            if (!newSortedRankings.containsKey(finalRank)) {
                newSortedRankings.put(finalRank, new HashSet<ResourceUnit>());
            }
            newSortedRankings.get(finalRank).add(entry.getKey());
            if (log.isDebugEnabled()) {
                log.debug("Added Resource Unit [{}] with Rank [{}]", entry.getKey().getResourceId(), finalRank);
            }
            // update metrics
            if (resourceUnit.getResourceId().contains(hostname)) {
                updateLoadBalancingMetrics(hostname, finalRank, ranking);
            }
        }
        updateBrokerToNamespaceToBundle();
        this.sortedRankings.set(newSortedRankings);
        this.resourceUnitRankings = newResourceUnitRankings;
    } else {
        log.info("Leader broker[{}] No ResourceUnits to rank this run, Using Old Ranking", pulsar.getWebServiceAddress());
    }
}

Example 19 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method doLoadShedding.

@Override
public void doLoadShedding() {
    long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
    long comfortLoadLevel = this.getLoadBalancerBrokerComfortLoadThresholdPercentage();
    log.info("Running load shedding task as leader broker, overload threshold {}, comfort loadlevel {}", overloadThreshold, comfortLoadLevel);
    // overloadedRU --> bundleName
    Map<ResourceUnit, String> namespaceBundlesToBeUnloaded = new HashMap<>();
    synchronized (currentLoadReports) {
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit overloadedRU = entry.getKey();
            LoadReport lr = entry.getValue();
            if (isAboveLoadLevel(lr.getSystemResourceUsage(), overloadThreshold)) {
                ResourceType bottleneckResourceType = lr.getBottleneckResourceType();
                Map<String, NamespaceBundleStats> bundleStats = lr.getSortedBundleStats(bottleneckResourceType);
                if (bundleStats == null) {
                    log.warn("Null bundle stats for bundle {}", lr.getName());
                    continue;
                }
                // 1. owns only one namespace
                if (bundleStats.size() == 1) {
                    // can't unload one namespace, just issue a warning message
                    String bundleName = lr.getBundleStats().keySet().iterator().next();
                    log.warn("HIGH USAGE WARNING : Sole namespace bundle {} is overloading broker {}. " + "No Load Shedding will be done on this broker", bundleName, overloadedRU.getResourceId());
                    continue;
                }
                for (Map.Entry<String, NamespaceBundleStats> bundleStat : bundleStats.entrySet()) {
                    String bundleName = bundleStat.getKey();
                    NamespaceBundleStats stats = bundleStat.getValue();
                    // We need at least one underloaded RU from list of candidates that can host this bundle
                    if (isBrokerAvailableForRebalancing(bundleStat.getKey(), comfortLoadLevel)) {
                        log.info("Namespace bundle {} will be unloaded from overloaded broker {}, bundle stats (topics: {}, producers {}, " + "consumers {}, bandwidthIn {}, bandwidthOut {})", bundleName, overloadedRU.getResourceId(), stats.topics, stats.producerCount, stats.consumerCount, stats.msgThroughputIn, stats.msgThroughputOut);
                        namespaceBundlesToBeUnloaded.put(overloadedRU, bundleName);
                    } else {
                        log.info("Unable to shed load from broker {}, no brokers with enough capacity available " + "for re-balancing {}", overloadedRU.getResourceId(), bundleName);
                    }
                    break;
                }
            }
        }
    }
    unloadNamespacesFromOverLoadedBrokers(namespaceBundlesToBeUnloaded);
}

Example 20 with LoadReport

use of org.apache.pulsar.policies.data.loadbalancer.LoadReport in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method isBrokerAvailableForRebalancing.

// todo: changeme: this can be optimized, we don't have to iterate through everytime
private boolean isBrokerAvailableForRebalancing(String bundleName, long maxLoadLevel) {
    NamespaceName namespaceName = NamespaceName.get(LoadManagerShared.getNamespaceNameFromBundleName(bundleName));
    Map<Long, Set<ResourceUnit>> availableBrokers = sortedRankings.get();
    // this does not have "http://" in front, hacky but no time to pretty up
    Multimap<Long, ResourceUnit> brokers = getFinalCandidates(namespaceName, availableBrokers);
    for (Object broker : brokers.values()) {
        ResourceUnit underloadedRU = (ResourceUnit) broker;
        LoadReport currentLoadReport = currentLoadReports.get(underloadedRU);
        if (isBelowLoadLevel(currentLoadReport.getSystemResourceUsage(), maxLoadLevel)) {
            return true;
        }
    }
    return false;
}