Search in sources :

Example 1 with ResourceUnitRanking

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method getLeastLoadedBroker.

private synchronized ResourceUnit getLeastLoadedBroker(ServiceUnitId serviceUnit, Map<Long, Set<ResourceUnit>> availableBrokers) {
    ResourceUnit selectedBroker = null;
    // If the broker is already assigned, return that candidate.
    for (final Map.Entry<ResourceUnit, ResourceUnitRanking> entry : resourceUnitRankings.entrySet()) {
        final ResourceUnit resourceUnit = entry.getKey();
        final ResourceUnitRanking ranking = entry.getValue();
        if (ranking.isServiceUnitPreAllocated(serviceUnit.toString())) {
            return resourceUnit;
        }
    }
    Multimap<Long, ResourceUnit> finalCandidates = getFinalCandidates(serviceUnit, availableBrokers);
    // Remove candidates that point to inactive brokers
    Set<String> activeBrokers = Collections.emptySet();
    try {
        activeBrokers = availableActiveBrokers.get();
        // Need to use an explicit Iterator object to prevent concurrent modification exceptions
        Iterator<Map.Entry<Long, ResourceUnit>> candidateIterator = finalCandidates.entries().iterator();
        while (candidateIterator.hasNext()) {
            Map.Entry<Long, ResourceUnit> candidate = candidateIterator.next();
            String candidateBrokerName = candidate.getValue().getResourceId().replace("http://", "");
            if (!activeBrokers.contains(candidateBrokerName)) {
                // Current candidate points to an inactive broker, so remove it
                candidateIterator.remove();
            }
        }
    } catch (Exception e) {
        log.warn("Error during attempt to remove inactive brokers while searching for least active broker", e);
    }
    if (finalCandidates.size() > 0) {
        if (this.getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LLS) || this.getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LEAST_MSG)) {
            selectedBroker = findBrokerForPlacement(finalCandidates, serviceUnit);
        } else {
            selectedBroker = placementStrategy.findBrokerForPlacement(finalCandidates);
        }
        log.info("Selected : [{}] for ServiceUnit : [{}]", selectedBroker.getResourceId(), serviceUnit.toString());
        return selectedBroker;
    } else {
        // No available broker found
        log.warn("No broker available to acquire service unit: [{}]", serviceUnit);
        return null;
    }
}
Also used : ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) Map(java.util.Map) HashMap(java.util.HashMap) TreeMap(java.util.TreeMap) ResourceUnitRanking(org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking) KeeperException(org.apache.zookeeper.KeeperException) IOException(java.io.IOException) PulsarServerException(org.apache.pulsar.broker.PulsarServerException)

Example 2 with ResourceUnitRanking

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method findBrokerForPlacement.

/**
 * Assign owner for specified ServiceUnit from the given candidates, following the the principles: 1) Optimum
 * distribution: fill up one broker till its load reaches optimum level (defined by underload threshold) before pull
 * another idle broker in; 2) Even distribution: once all brokers' load are above optimum level, maintain all
 * brokers to have even load; 3) Set the underload threshold to small value (like 1) for pure even distribution, and
 * high value (like 80) for pure optimum distribution;
 *
 * Strategy to select broker: 1) The first choice is the least loaded broker which is underload but not idle; 2) The
 * second choice is idle broker (if there is any); 3) Othewise simply select the least loaded broker if it is NOT
 * overloaded; 4) If all brokers are overloaded, select the broker with maximum available capacity (considering
 * brokers could have different hardware configuration, this usually means to select the broker with more hardware
 * resource);
 *
 * Broker's load level: 1) Load ranking (triggered by LoadReport update) estimate the load level according to the
 * resourse usage and namespace bundles already loaded by each broker; 2) When leader broker decide the owner for a
 * new namespace bundle, it may take time for the real owner to actually load the bundle and refresh LoadReport,
 * leader broker will store the bundle in a list called preAllocatedBundles, and the quota of all
 * preAllocatedBundles in preAllocatedQuotas, and re-estimate the broker's load level by putting the
 * preAllocatedQuota into calculation; 3) Everything (preAllocatedBundles and preAllocatedQuotas) will get reset in
 * load ranking.
 */
private synchronized ResourceUnit findBrokerForPlacement(Multimap<Long, ResourceUnit> candidates, ServiceUnitId serviceUnit) {
    long underloadThreshold = this.getLoadBalancerBrokerUnderloadedThresholdPercentage();
    long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
    ResourceQuota defaultQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
    double minLoadPercentage = 101.0;
    long maxAvailability = -1;
    ResourceUnit idleRU = null;
    ResourceUnit maxAvailableRU = null;
    ResourceUnit randomRU = null;
    ResourceUnit selectedRU = null;
    ResourceUnitRanking selectedRanking = null;
    String serviceUnitId = serviceUnit.toString();
    // If the ranking is expected to be in the range [0,100] (which is the case for LOADBALANCER_STRATEGY_LLS),
    // the ranks are bounded. Otherwise (as is the case in LOADBALANCER_STRATEGY_LEAST_MSG, the ranks are simply
    // the total message rate which is in the range [0,Infinity) so they are unbounded. The
    // "boundedness" affects how two ranks are compared to see which one is better
    boolean unboundedRanks = getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LEAST_MSG);
    long randomBrokerIndex = (candidates.size() > 0) ? (this.brokerRotationCursor % candidates.size()) : 0;
    // find the least loaded & not-idle broker
    for (Map.Entry<Long, ResourceUnit> candidateOwner : candidates.entries()) {
        ResourceUnit candidate = candidateOwner.getValue();
        randomBrokerIndex--;
        // skip broker which is not ranked. this should never happen except in unit test
        if (!resourceUnitRankings.containsKey(candidate)) {
            continue;
        }
        String resourceUnitId = candidate.getResourceId();
        ResourceUnitRanking ranking = resourceUnitRankings.get(candidate);
        // check if this ServiceUnit is already loaded
        if (ranking.isServiceUnitLoaded(serviceUnitId)) {
            ranking.removeLoadedServiceUnit(serviceUnitId, this.getResourceQuota(serviceUnitId));
        }
        // record a random broker
        if (randomBrokerIndex < 0 && randomRU == null) {
            randomRU = candidate;
        }
        // check the available capacity
        double loadPercentage = ranking.getEstimatedLoadPercentage();
        double availablePercentage = Math.max(0, (100 - loadPercentage) / 100);
        long availability = (long) (ranking.estimateMaxCapacity(defaultQuota) * availablePercentage);
        if (availability > maxAvailability) {
            maxAvailability = availability;
            maxAvailableRU = candidate;
        }
        // check the load percentage
        if (ranking.isIdle()) {
            if (idleRU == null) {
                idleRU = candidate;
            }
        } else {
            if (selectedRU == null) {
                selectedRU = candidate;
                selectedRanking = ranking;
                minLoadPercentage = loadPercentage;
            } else {
                if ((unboundedRanks ? ranking.compareMessageRateTo(selectedRanking) : ranking.compareTo(selectedRanking)) < 0) {
                    minLoadPercentage = loadPercentage;
                    selectedRU = candidate;
                    selectedRanking = ranking;
                }
            }
        }
    }
    if ((minLoadPercentage > underloadThreshold && idleRU != null) || selectedRU == null) {
        // assigned to idle broker is the least loaded broker already have optimum load (which means NOT
        // underloaded), or all brokers are idle
        selectedRU = idleRU;
    } else if (minLoadPercentage >= 100.0 && randomRU != null && !unboundedRanks) {
        // all brokers are full, assign to a random one
        selectedRU = randomRU;
    } else if (minLoadPercentage > overloadThreshold && !unboundedRanks) {
        // assign to the broker with maximum available capacity if all brokers are overloaded
        selectedRU = maxAvailableRU;
    }
    // re-calculate load level for selected broker
    if (selectedRU != null) {
        this.brokerRotationCursor = (this.brokerRotationCursor + 1) % 1000000;
        ResourceUnitRanking ranking = resourceUnitRankings.get(selectedRU);
        String loadPercentageDesc = ranking.getEstimatedLoadPercentageString();
        log.info("Assign {} to {} with ({}).", serviceUnitId, selectedRU.getResourceId(), loadPercentageDesc);
        if (!ranking.isServiceUnitPreAllocated(serviceUnitId)) {
            final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(serviceUnitId);
            final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(serviceUnitId);
            ResourceQuota quota = this.getResourceQuota(serviceUnitId);
            // Add preallocated bundle range so incoming bundles from the same namespace are not assigned to the
            // same broker.
            brokerToNamespaceToBundleRange.computeIfAbsent(selectedRU.getResourceId().replace("http://", ""), k -> new HashMap<>()).computeIfAbsent(namespaceName, k -> new HashSet<>()).add(bundleRange);
            ranking.addPreAllocatedServiceUnit(serviceUnitId, quota);
            resourceUnitRankings.put(selectedRU, ranking);
        }
    }
    return selectedRU;
}
Also used : CreateMode(org.apache.zookeeper.CreateMode) Metrics(org.apache.pulsar.common.stats.Metrics) LoadingCache(com.google.common.cache.LoadingCache) DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) Ids(org.apache.zookeeper.ZooDefs.Ids) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) LoggerFactory(org.slf4j.LoggerFactory) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) Stat(org.apache.zookeeper.data.Stat) SystemResourceUsage(org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage) TreeMultimap(com.google.common.collect.TreeMultimap) Map(java.util.Map) ZooKeeperDataCache(org.apache.pulsar.zookeeper.ZooKeeperDataCache) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) AdminResource.jsonMapper(org.apache.pulsar.broker.admin.AdminResource.jsonMapper) BrokerHostUsage(org.apache.pulsar.broker.loadbalance.BrokerHostUsage) Set(java.util.Set) ZooKeeperChildrenCache(org.apache.pulsar.zookeeper.ZooKeeperChildrenCache) ZooKeeperCacheListener(org.apache.pulsar.zookeeper.ZooKeeperCacheListener) LoadReport(org.apache.pulsar.policies.data.loadbalancer.LoadReport) Executors(java.util.concurrent.Executors) ZkUtils(org.apache.bookkeeper.util.ZkUtils) Sets(com.google.common.collect.Sets) CacheLoader(com.google.common.cache.CacheLoader) List(java.util.List) ServiceUnitId(org.apache.pulsar.common.naming.ServiceUnitId) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) CacheBuilder(com.google.common.cache.CacheBuilder) LocalDateTime(java.time.LocalDateTime) HashMap(java.util.HashMap) ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) LOAD_REPORT_UPDATE_MIMIMUM_INTERVAL(org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.LOAD_REPORT_UPDATE_MIMIMUM_INTERVAL) Multimap(com.google.common.collect.Multimap) AtomicReference(java.util.concurrent.atomic.AtomicReference) StringUtils.isNotEmpty(org.apache.commons.lang3.StringUtils.isNotEmpty) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Lists(com.google.common.collect.Lists) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Charsets(com.google.common.base.Charsets) ResourceQuota(org.apache.pulsar.common.policies.data.ResourceQuota) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) KeeperException(org.apache.zookeeper.KeeperException) SystemUtils(org.apache.commons.lang3.SystemUtils) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) BrokerTopicLoadingPredicate(org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate) IOException(java.io.IOException) PulsarService(org.apache.pulsar.broker.PulsarService) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) PlacementStrategy(org.apache.pulsar.broker.loadbalance.PlacementStrategy) ResourceType(org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage.ResourceType) TreeMap(java.util.TreeMap) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Deserializer(org.apache.pulsar.zookeeper.ZooKeeperCache.Deserializer) ResourceUnitRanking(org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking) Collections(java.util.Collections) ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) ResourceQuota(org.apache.pulsar.common.policies.data.ResourceQuota) ResourceUnitRanking(org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking) Map(java.util.Map) HashMap(java.util.HashMap) TreeMap(java.util.TreeMap) HashSet(java.util.HashSet)

Example 3 with ResourceUnitRanking

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testPrimary.

@Test(enabled = true)
public void testPrimary() throws Exception {
    createNamespacePolicies(pulsar1);
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort(), rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    // inject the load report and rankings
    Map<ResourceUnit, org.apache.pulsar.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport.setSystemResourceUsage(new SystemResourceUsage());
    loadReports.put(ru1, loadReport);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), new HashSet<String>(), new ResourceQuota(), new HashSet<String>(), new ResourceQuota());
    Map<ResourceUnit, ResourceUnitRanking> rankings = new HashMap<>();
    rankings.put(ru1, ranking);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "resourceUnitRankings", rankings);
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "sortedRankings", sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
    // broker is not active so found should be null
    assertNotEquals(found, null, "did not find a broker when expected one to be found");
}
Also used : PulsarResourceDescription(org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription) HashMap(java.util.HashMap) ResourceAvailabilityRanker(org.apache.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) ResourceQuota(org.apache.pulsar.common.policies.data.ResourceQuota) ResourceUnitRanking(org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking) HashSet(java.util.HashSet) SystemResourceUsage(org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage) ResourceUsage(org.apache.pulsar.policies.data.loadbalancer.ResourceUsage) SystemResourceUsage(org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage) AtomicReference(java.util.concurrent.atomic.AtomicReference) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.testng.annotations.Test)

Example 4 with ResourceUnitRanking

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method doLoadRanking.

/**
 * Rank brokers by available capacity, or load percentage, based on placement strategy:
 *
 * - Available capacity for weighted random selection (weightedRandomSelection): ranks ResourceUnits units based on
 * estimation of their capacity which is basically how many bundles each ResourceUnit is able can handle with its
 * available resources (CPU, memory, network, etc);
 *
 * - Load percentage for least loaded server (leastLoadedServer): ranks ResourceUnits units based on estimation of
 * their load percentage which is basically how many percent of resource is allocated which is
 * max(resource_actually_used, resource_quota)
 *
 * If we fail to collect the Load Reports OR fail to process them for the first time, it means the leader does not
 * have enough information to make a decision so we set it to ready when we collect and process the load reports
 * successfully the first time.
 */
private synchronized void doLoadRanking() {
    ResourceUnitRanking.setCpuUsageByMsgRate(this.realtimeCpuLoadFactor);
    String hostname = pulsar.getAdvertisedAddress();
    String strategy = this.getLoadBalancerPlacementStrategy();
    log.info("doLoadRanking - load balancing strategy: {}", strategy);
    if (!currentLoadReports.isEmpty()) {
        Map<Long, Set<ResourceUnit>> newSortedRankings = Maps.newTreeMap();
        Map<ResourceUnit, ResourceUnitRanking> newResourceUnitRankings = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            // calculate rankings
            Set<String> loadedBundles = loadReport.getBundles();
            Set<String> preAllocatedBundles = null;
            if (resourceUnitRankings.containsKey(resourceUnit)) {
                preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles();
                preAllocatedBundles.removeAll(loadedBundles);
            } else {
                preAllocatedBundles = new HashSet<>();
            }
            ResourceQuota allocatedQuota = getTotalAllocatedQuota(loadedBundles);
            ResourceQuota preAllocatedQuota = getTotalAllocatedQuota(preAllocatedBundles);
            ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), loadedBundles, allocatedQuota, preAllocatedBundles, preAllocatedQuota);
            newResourceUnitRankings.put(resourceUnit, ranking);
            // generated sorted ranking
            double loadPercentage = ranking.getEstimatedLoadPercentage();
            long maxCapacity = ranking.estimateMaxCapacity(pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota());
            long finalRank = 0;
            if (strategy.equals(LOADBALANCER_STRATEGY_LLS)) {
                finalRank = (long) loadPercentage;
            } else if (strategy.equals(LOADBALANCER_STRATEGY_LEAST_MSG)) {
                finalRank = (long) ranking.getEstimatedMessageRate();
            } else {
                double idleRatio = (100 - loadPercentage) / 100;
                finalRank = (long) (maxCapacity * idleRatio * idleRatio);
            }
            if (!newSortedRankings.containsKey(finalRank)) {
                newSortedRankings.put(finalRank, new HashSet<ResourceUnit>());
            }
            newSortedRankings.get(finalRank).add(entry.getKey());
            if (log.isDebugEnabled()) {
                log.debug("Added Resource Unit [{}] with Rank [{}]", entry.getKey().getResourceId(), finalRank);
            }
            // update metrics
            if (resourceUnit.getResourceId().contains(hostname)) {
                updateLoadBalancingMetrics(hostname, finalRank, ranking);
            }
        }
        updateBrokerToNamespaceToBundle();
        this.sortedRankings.set(newSortedRankings);
        this.resourceUnitRankings = newResourceUnitRankings;
    } else {
        log.info("Leader broker[{}] No ResourceUnits to rank this run, Using Old Ranking", pulsar.getWebServiceAddress());
    }
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) ResourceUnit(org.apache.pulsar.broker.loadbalance.ResourceUnit) ResourceQuota(org.apache.pulsar.common.policies.data.ResourceQuota) LoadReport(org.apache.pulsar.policies.data.loadbalancer.LoadReport) ResourceUnitRanking(org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking) Map(java.util.Map) HashMap(java.util.HashMap) TreeMap(java.util.TreeMap)

Aggregations

HashMap (java.util.HashMap)4 Map (java.util.Map)4 ResourceUnitRanking (org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking)4 HashSet (java.util.HashSet)3 TreeMap (java.util.TreeMap)3 ResourceUnit (org.apache.pulsar.broker.loadbalance.ResourceUnit)3 ResourceQuota (org.apache.pulsar.common.policies.data.ResourceQuota)3 IOException (java.io.IOException)2 Set (java.util.Set)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2 PulsarServerException (org.apache.pulsar.broker.PulsarServerException)2 SystemResourceUsage (org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage)2 KeeperException (org.apache.zookeeper.KeeperException)2 Charsets (com.google.common.base.Charsets)1 CacheBuilder (com.google.common.cache.CacheBuilder)1 CacheLoader (com.google.common.cache.CacheLoader)1 LoadingCache (com.google.common.cache.LoadingCache)1 Lists (com.google.common.collect.Lists)1 Maps (com.google.common.collect.Maps)1 Multimap (com.google.common.collect.Multimap)1