Examples with ResourceUnit com.yahoo.pulsar.broker.loadbalance.ResourceUnit used on opensource projects

Example 6 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method doLoadShedding.

@Override
public void doLoadShedding() {
    long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
    long comfortLoadLevel = this.getLoadBalancerBrokerComfortLoadThresholdPercentage();
    log.info("Running load shedding task as leader broker, overload threshold {}, comfort loadlevel {}", overloadThreshold, comfortLoadLevel);
    // overloadedRU --> bundleName
    Map<ResourceUnit, String> namespaceBundlesToBeUnloaded = new HashMap<>();
    synchronized (currentLoadReports) {
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit overloadedRU = entry.getKey();
            LoadReport lr = entry.getValue();
            if (isAboveLoadLevel(lr.getSystemResourceUsage(), overloadThreshold)) {
                ResourceType bottleneckResourceType = lr.getBottleneckResourceType();
                Map<String, NamespaceBundleStats> bundleStats = lr.getSortedBundleStats(bottleneckResourceType);
                // 1. owns only one namespace
                if (bundleStats.size() == 1) {
                    // can't unload one namespace, just issue a warning message
                    String bundleName = lr.getBundleStats().keySet().iterator().next();
                    log.warn("HIGH USAGE WARNING : Sole namespace bundle {} is overloading broker {}. " + "No Load Shedding will be done on this broker", bundleName, overloadedRU.getResourceId());
                    continue;
                }
                for (Map.Entry<String, NamespaceBundleStats> bundleStat : bundleStats.entrySet()) {
                    String bundleName = bundleStat.getKey();
                    NamespaceBundleStats stats = bundleStat.getValue();
                    // We need at least one underloaded RU from list of candidates that can host this bundle
                    if (isBrokerAvailableForRebalancing(bundleStat.getKey(), comfortLoadLevel)) {
                        log.info("Namespace bundle {} will be unloaded from overloaded broker {}, bundle stats (topics: {}, producers {}, " + "consumers {}, bandwidthIn {}, bandwidthOut {})", bundleName, overloadedRU.getResourceId(), stats.topics, stats.producerCount, stats.consumerCount, stats.msgThroughputIn, stats.msgThroughputOut);
                        namespaceBundlesToBeUnloaded.put(overloadedRU, bundleName);
                    } else {
                        log.info("Unable to shed load from broker {}, no brokers with enough capacity available " + "for re-balancing {}", overloadedRU.getResourceId(), bundleName);
                    }
                    break;
                }
            }
        }
    }
    unloadNamespacesFromOverLoadedBrokers(namespaceBundlesToBeUnloaded);
}

Example 7 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method getLeastLoadedBroker.

private ResourceUnit getLeastLoadedBroker(ServiceUnitId serviceUnit, Map<Long, Set<ResourceUnit>> availableBrokers) {
    ResourceUnit selectedBroker = null;
    Multimap<Long, ResourceUnit> finalCandidates = getFinalCandidates(serviceUnit, availableBrokers);
    // Remove candidates that point to inactive brokers
    Set<String> activeBrokers = Collections.emptySet();
    try {
        activeBrokers = availableActiveBrokers.get();
        // Need to use an explicit Iterator object to prevent concurrent modification exceptions
        Iterator<Map.Entry<Long, ResourceUnit>> candidateIterator = finalCandidates.entries().iterator();
        while (candidateIterator.hasNext()) {
            Map.Entry<Long, ResourceUnit> candidate = candidateIterator.next();
            String candidateBrokerName = candidate.getValue().getResourceId().replace("http://", "");
            if (!activeBrokers.contains(candidateBrokerName)) {
                // Current candidate points to an inactive broker, so remove it
                candidateIterator.remove();
            }
        }
    } catch (Exception e) {
        log.warn("Error during attempt to remove inactive brokers while searching for least active broker", e);
    }
    if (finalCandidates.size() > 0) {
        if (this.getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LLS)) {
            selectedBroker = findBrokerForPlacement(finalCandidates, serviceUnit);
        } else {
            selectedBroker = placementStrategy.findBrokerForPlacement(finalCandidates);
        }
        log.debug("Selected : [{}] for ServiceUnit : [{}]", selectedBroker.getResourceId(), serviceUnit.getNamespaceObject().toString());
        return selectedBroker;
    } else {
        // No available broker found
        log.warn("No broker available to acquire service unit: [{}]", serviceUnit);
        return null;
    }
}

Example 8 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method getFinalCandidates.

private Multimap<Long, ResourceUnit> getFinalCandidates(ServiceUnitId serviceUnit, Map<Long, Set<ResourceUnit>> availableBrokers) {
    // need multimap or at least set of RUs
    Multimap<Long, ResourceUnit> matchedPrimaries = TreeMultimap.create();
    Multimap<Long, ResourceUnit> matchedShared = TreeMultimap.create();
    NamespaceName namespace = serviceUnit.getNamespaceObject();
    boolean isIsolationPoliciesPresent = policies.IsIsolationPoliciesPresent(namespace);
    if (isIsolationPoliciesPresent) {
        log.debug("Isolation Policies Present for namespace - [{}]", namespace.toString());
    }
    for (Map.Entry<Long, Set<ResourceUnit>> entry : availableBrokers.entrySet()) {
        for (ResourceUnit ru : entry.getValue()) {
            log.debug("Considering Resource Unit [{}] with Rank [{}] for serviceUnit [{}]", ru.getResourceId(), entry.getKey(), serviceUnit);
            URL brokerUrl = null;
            try {
                brokerUrl = new URL(String.format(ru.getResourceId()));
            } catch (MalformedURLException e) {
                log.error("Unable to parse brokerUrl from ResourceUnitId - [{}]", e);
                continue;
            }
            // todo: in future check if the resource unit has resources to take the namespace
            if (isIsolationPoliciesPresent) {
                // note: serviceUnitID is namespace name and ResourceID is brokerName
                if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) {
                    matchedPrimaries.put(entry.getKey(), ru);
                    if (log.isDebugEnabled()) {
                        log.debug("Added Primary Broker - [{}] as possible Candidates for" + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString());
                    }
                } else if (policies.isSharedBroker(brokerUrl.getHost())) {
                    matchedShared.put(entry.getKey(), ru);
                    if (log.isDebugEnabled()) {
                        log.debug("Added Shared Broker - [{}] as possible " + "Candidates for namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString());
                    }
                } else {
                    if (log.isDebugEnabled()) {
                        log.debug("Skipping Broker - [{}] not primary broker and not shared" + " for namespace - [{}] ", brokerUrl.getHost(), namespace.toString());
                    }
                }
            } else {
                if (policies.isSharedBroker(brokerUrl.getHost())) {
                    matchedShared.put(entry.getKey(), ru);
                    log.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", brokerUrl.getHost(), namespace.toString());
                }
            }
        }
    }
    if (isIsolationPoliciesPresent) {
        return getFinalCandidatesWithPolicy(namespace, matchedPrimaries, matchedShared);
    } else {
        log.debug("Policies not present for namespace - [{}] so only " + "considering shared [{}] brokers for possible owner", namespace.toString(), matchedShared.size());
        return getFinalCandidatesNoPolicy(matchedShared);
    }
}

Example 9 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class WRRPlacementStrategy method findBrokerForPlacement.

/**
     * <code>
     * Function : getByWeightedRoundRobin
     *            returns ResourceUnit selected by WRR algorithm based on available resource on RU
     * ^
     * |
     * |
     * |
     * |                |                        |                                |     |
     * |                |                        |                                |     |
     * |   Broker 2     |       Broker 3         |         Broker 1               |  B4 |
     * |                |                        |                                |     |
     * +----------------+------------------------+--------------------------------+--------->
     * 0                20                       50                               90    100
     *
     * This is weighted Round robin, we calculate weight based on availability of resources;
     * total availability is taken as a full range then each broker is given range based on
     *  its resource availability, if the number generated within total range happens to be in
     * broker's range, that broker is selected
     * </code>
     */
public ResourceUnit findBrokerForPlacement(Multimap<Long, ResourceUnit> finalCandidates) {
    if (finalCandidates.isEmpty()) {
        return null;
    }
    log.debug("Total Final Candidates selected - [{}]", finalCandidates.size());
    int totalAvailability = 0;
    for (Map.Entry<Long, ResourceUnit> candidateOwner : finalCandidates.entries()) {
        totalAvailability += candidateOwner.getKey().intValue();
    }
    ResourceUnit selectedRU = null;
    if (totalAvailability <= 0) {
        // for now, pick anyone and return that one, because when we don't have ranking we put O for each broker
        return Iterables.get(finalCandidates.get(0L), rand.nextInt(finalCandidates.size()));
    }
    int weightedSelector = rand.nextInt(totalAvailability);
    log.debug("Generated Weighted Selector Number - [{}] ", weightedSelector);
    int weightRangeSoFar = 0;
    for (Map.Entry<Long, ResourceUnit> candidateOwner : finalCandidates.entries()) {
        weightRangeSoFar += candidateOwner.getKey();
        if (weightedSelector < weightRangeSoFar) {
            selectedRU = candidateOwner.getValue();
            log.debug(" Weighted Round Robin Selected RU - [{}]", candidateOwner.getValue().getResourceId());
            break;
        }
    }
    return selectedRU;
}

Example 10 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method findBrokerForPlacement.

/**
     * Assign owner for specified ServiceUnit from the given candidates, following the the principles: 1) Optimum
     * distribution: fill up one broker till its load reaches optimum level (defined by underload threshold) before pull
     * another idle broker in; 2) Even distribution: once all brokers' load are above optimum level, maintain all
     * brokers to have even load; 3) Set the underload threshold to small value (like 1) for pure even distribution, and
     * high value (like 80) for pure optimum distribution;
     *
     * Strategy to select broker: 1) The first choice is the least loaded broker which is underload but not idle; 2) The
     * second choice is idle broker (if there is any); 3) Othewise simply select the least loaded broker if it is NOT
     * overloaded; 4) If all brokers are overloaded, select the broker with maximum available capacity (considering
     * brokers could have different hardware configuration, this usually means to select the broker with more hardware
     * resource);
     *
     * Broker's load level: 1) Load ranking (triggered by LoadReport update) estimate the load level according to the
     * resourse usage and namespace bundles already loaded by each broker; 2) When leader broker decide the owner for a
     * new namespace bundle, it may take time for the real owner to actually load the bundle and refresh LoadReport,
     * leader broker will store the bundle in a list called preAllocatedBundles, and the quota of all
     * preAllocatedBundles in preAllocatedQuotas, and re-estimate the broker's load level by putting the
     * preAllocatedQuota into calculation; 3) Everything (preAllocatedBundles and preAllocatedQuotas) will get reset in
     * load ranking.
     */
private ResourceUnit findBrokerForPlacement(Multimap<Long, ResourceUnit> candidates, ServiceUnitId serviceUnit) {
    long underloadThreshold = this.getLoadBalancerBrokerUnderloadedThresholdPercentage();
    long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
    ResourceQuota defaultQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
    double minLoadPercentage = 101.0;
    long maxAvailability = -1;
    ResourceUnit idleRU = null;
    ResourceUnit maxAvailableRU = null;
    ResourceUnit randomRU = null;
    ResourceUnit selectedRU = null;
    ResourceUnitRanking selectedRanking = null;
    String serviceUnitId = serviceUnit.toString();
    synchronized (resourceUnitRankings) {
        long randomBrokerIndex = (candidates.size() > 0) ? (this.brokerRotationCursor % candidates.size()) : 0;
        // find the least loaded & not-idle broker
        for (Map.Entry<Long, ResourceUnit> candidateOwner : candidates.entries()) {
            ResourceUnit candidate = candidateOwner.getValue();
            randomBrokerIndex--;
            // skip broker which is not ranked. this should never happen except in unit test
            if (!resourceUnitRankings.containsKey(candidate)) {
                continue;
            }
            // check if this ServiceUnit is already pre-allocated
            String resourceUnitId = candidate.getResourceId();
            ResourceUnitRanking ranking = resourceUnitRankings.get(candidate);
            if (ranking.isServiceUnitPreAllocated(serviceUnitId)) {
                return candidate;
            }
            // check if this ServiceUnit is already loaded
            if (ranking.isServiceUnitLoaded(serviceUnitId)) {
                ranking.removeLoadedServiceUnit(serviceUnitId, this.getResourceQuota(serviceUnitId));
            }
            // record a random broker
            if (randomBrokerIndex < 0 && randomRU == null) {
                randomRU = candidate;
            }
            // check the available capacity
            double loadPercentage = ranking.getEstimatedLoadPercentage();
            double availablePercentage = Math.max(0, (100 - loadPercentage) / 100);
            long availability = (long) (ranking.estimateMaxCapacity(defaultQuota) * availablePercentage);
            if (availability > maxAvailability) {
                maxAvailability = availability;
                maxAvailableRU = candidate;
            }
            // check the load percentage
            if (ranking.isIdle()) {
                if (idleRU == null) {
                    idleRU = candidate;
                }
            } else {
                if (selectedRU == null) {
                    selectedRU = candidate;
                    selectedRanking = ranking;
                    minLoadPercentage = loadPercentage;
                } else {
                    if (ranking.compareTo(selectedRanking) < 0) {
                        minLoadPercentage = loadPercentage;
                        selectedRU = candidate;
                        selectedRanking = ranking;
                    }
                }
            }
        }
        if ((minLoadPercentage > underloadThreshold && idleRU != null) || selectedRU == null) {
            // assigned to idle broker is the least loaded broker already have optimum load (which means NOT
            // underloaded), or all brokers are idle
            selectedRU = idleRU;
        } else if (minLoadPercentage >= 100.0 && randomRU != null) {
            // all brokers are full, assign to a random one
            selectedRU = randomRU;
        } else if (minLoadPercentage > overloadThreshold) {
            // assign to the broker with maximum available capacity if all brokers are overloaded
            selectedRU = maxAvailableRU;
        }
        // re-calculate load level for selected broker
        if (selectedRU != null) {
            this.brokerRotationCursor = (this.brokerRotationCursor + 1) % 1000000;
            ResourceUnitRanking ranking = resourceUnitRankings.get(selectedRU);
            String loadPercentageDesc = ranking.getEstimatedLoadPercentageString();
            log.info("Assign {} to {} with ({}).", serviceUnitId, selectedRU.getResourceId(), loadPercentageDesc);
            if (!ranking.isServiceUnitPreAllocated(serviceUnitId)) {
                ResourceQuota quota = this.getResourceQuota(serviceUnitId);
                ranking.addPreAllocatedServiceUnit(serviceUnitId, quota);
            }
        }
    }
    return selectedRU;
}