use of org.apache.pulsar.broker.loadbalance.ResourceUnit in project incubator-pulsar by apache.
the class SimpleLoadManagerImpl method getLeastLoadedBroker.
private synchronized ResourceUnit getLeastLoadedBroker(ServiceUnitId serviceUnit, Map<Long, Set<ResourceUnit>> availableBrokers) {
ResourceUnit selectedBroker = null;
// If the broker is already assigned, return that candidate.
for (final Map.Entry<ResourceUnit, ResourceUnitRanking> entry : resourceUnitRankings.entrySet()) {
final ResourceUnit resourceUnit = entry.getKey();
final ResourceUnitRanking ranking = entry.getValue();
if (ranking.isServiceUnitPreAllocated(serviceUnit.toString())) {
return resourceUnit;
}
}
Multimap<Long, ResourceUnit> finalCandidates = getFinalCandidates(serviceUnit, availableBrokers);
// Remove candidates that point to inactive brokers
Set<String> activeBrokers = Collections.emptySet();
try {
activeBrokers = availableActiveBrokers.get();
// Need to use an explicit Iterator object to prevent concurrent modification exceptions
Iterator<Map.Entry<Long, ResourceUnit>> candidateIterator = finalCandidates.entries().iterator();
while (candidateIterator.hasNext()) {
Map.Entry<Long, ResourceUnit> candidate = candidateIterator.next();
String candidateBrokerName = candidate.getValue().getResourceId().replace("http://", "");
if (!activeBrokers.contains(candidateBrokerName)) {
// Current candidate points to an inactive broker, so remove it
candidateIterator.remove();
}
}
} catch (Exception e) {
log.warn("Error during attempt to remove inactive brokers while searching for least active broker", e);
}
if (finalCandidates.size() > 0) {
if (this.getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LLS) || this.getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LEAST_MSG)) {
selectedBroker = findBrokerForPlacement(finalCandidates, serviceUnit);
} else {
selectedBroker = placementStrategy.findBrokerForPlacement(finalCandidates);
}
log.info("Selected : [{}] for ServiceUnit : [{}]", selectedBroker.getResourceId(), serviceUnit.toString());
return selectedBroker;
} else {
// No available broker found
log.warn("No broker available to acquire service unit: [{}]", serviceUnit);
return null;
}
}
use of org.apache.pulsar.broker.loadbalance.ResourceUnit in project incubator-pulsar by apache.
the class SimpleLoadManagerImpl method updateRanking.
private void updateRanking() {
try {
synchronized (currentLoadReports) {
currentLoadReports.clear();
Set<String> activeBrokers = availableActiveBrokers.get();
for (String broker : activeBrokers) {
try {
String key = String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, broker);
LoadReport lr = loadReportCacheZk.get(key).orElseThrow(() -> new KeeperException.NoNodeException());
ResourceUnit ru = new SimpleResourceUnit(String.format("http://%s", lr.getName()), fromLoadReport(lr));
this.currentLoadReports.put(ru, lr);
} catch (Exception e) {
log.warn("Error reading load report from Cache for broker - [{}], [{}]", broker, e);
}
}
updateRealtimeResourceQuota();
doLoadRanking();
}
} catch (Exception e) {
log.warn("Error reading active brokers list from zookeeper while re-ranking load reports [{}]", e);
}
}
use of org.apache.pulsar.broker.loadbalance.ResourceUnit in project incubator-pulsar by apache.
the class SimpleLoadManagerImpl method updateRealtimeResourceQuota.
private synchronized void updateRealtimeResourceQuota() {
long memObjectGroupSize = 500;
if (!currentLoadReports.isEmpty()) {
long totalBundles = 0;
long totalMemGroups = 0;
double totalMsgRateIn = 0.0;
double totalMsgRateOut = 0.0;
double totalMsgRate = 0.0;
double totalCpuUsage = 0.0;
double totalMemoryUsage = 0.0;
double totalBandwidthIn = 0.0;
double totalBandwidthOut = 0.0;
long loadReportTimestamp = -1;
// update resource factors
for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
LoadReport loadReport = entry.getValue();
if (loadReport.getTimestamp() > loadReportTimestamp) {
loadReportTimestamp = loadReport.getTimestamp();
}
Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
if (bundleStats == null) {
continue;
}
for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
totalBundles++;
NamespaceBundleStats stats = statsEntry.getValue();
totalMemGroups += (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
totalBandwidthIn += stats.msgThroughputIn;
totalBandwidthOut += stats.msgThroughputOut;
}
SystemResourceUsage resUsage = loadReport.getSystemResourceUsage();
totalMsgRateIn += loadReport.getMsgRateIn();
totalMsgRateOut += loadReport.getMsgRateOut();
totalCpuUsage = totalCpuUsage + resUsage.getCpu().usage;
totalMemoryUsage = totalMemoryUsage + resUsage.getMemory().usage;
}
totalMsgRate = totalMsgRateIn + totalMsgRateOut;
long timePast = loadReportTimestamp - this.lastResourceQuotaUpdateTimestamp;
this.lastResourceQuotaUpdateTimestamp = loadReportTimestamp;
if (totalMsgRate > 1000 && totalMemGroups > 30) {
this.realtimeCpuLoadFactor = timeSmoothValue(this.realtimeCpuLoadFactor, totalCpuUsage / totalMsgRate, RESOURCE_QUOTA_MIN_CPU_FACTOR, RESOURCE_QUOTA_MAX_CPU_FACTOR, timePast);
this.realtimeMemoryLoadFactor = timeSmoothValue(this.realtimeMemoryLoadFactor, totalMemoryUsage / totalMemGroups, RESOURCE_QUOTA_MIN_MEM_FACTOR, RESOURCE_QUOTA_MAX_MEM_FACTOR, timePast);
}
// calculate average bundle
if (totalBundles > 30 && this.realtimeAvgResourceQuota.getDynamic()) {
ResourceQuota oldQuota = this.realtimeAvgResourceQuota;
ResourceQuota newQuota = timeSmoothQuota(oldQuota, totalMsgRateIn / totalBundles, totalMsgRateOut / totalBundles, totalBandwidthIn / totalBundles, totalBandwidthOut / totalBundles, totalMemoryUsage / totalBundles, timePast);
this.realtimeAvgResourceQuota = newQuota;
}
// update realtime quota for each bundle
Map<String, ResourceQuota> newQuotas = new HashMap<>();
for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
ResourceUnit resourceUnit = entry.getKey();
LoadReport loadReport = entry.getValue();
Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
if (bundleStats == null) {
continue;
}
for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
String bundle = statsEntry.getKey();
NamespaceBundleStats stats = statsEntry.getValue();
long memGroupCount = (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
double newMemoryQuota = memGroupCount * this.realtimeMemoryLoadFactor;
ResourceQuota oldQuota = getResourceQuota(bundle);
ResourceQuota newQuota = timeSmoothQuota(oldQuota, stats.msgRateIn, stats.msgRateOut, stats.msgThroughputIn, stats.msgThroughputOut, newMemoryQuota, timePast);
newQuotas.put(bundle, newQuota);
}
}
this.realtimeResourceQuotas.set(newQuotas);
}
}
use of org.apache.pulsar.broker.loadbalance.ResourceUnit in project incubator-pulsar by apache.
the class SimpleLoadManagerImpl method getAvailableBrokers.
private Map<Long, Set<ResourceUnit>> getAvailableBrokers(ServiceUnitId serviceUnitId) throws Exception {
Map<Long, Set<ResourceUnit>> availableBrokers = sortedRankings.get();
if (availableBrokers.isEmpty()) {
// Create a map with all available brokers with no load information
Set<String> activeBrokers = availableActiveBrokers.get(LOADBALANCE_BROKERS_ROOT);
List<String> brokersToShuffle = new ArrayList<>(activeBrokers);
Collections.shuffle(brokersToShuffle);
activeBrokers = new HashSet<>(brokersToShuffle);
availableBrokers = Maps.newTreeMap();
for (String broker : activeBrokers) {
ResourceUnit resourceUnit = new SimpleResourceUnit(String.format("http://%s", broker), new PulsarResourceDescription());
availableBrokers.computeIfAbsent(0L, key -> Sets.newTreeSet()).add(resourceUnit);
}
log.info("Choosing at random from broker list: [{}]", availableBrokers.values());
}
return availableBrokers;
}
use of org.apache.pulsar.broker.loadbalance.ResourceUnit in project incubator-pulsar by apache.
the class SimpleLoadManagerImpl method findBrokerForPlacement.
/**
* Assign owner for specified ServiceUnit from the given candidates, following the the principles: 1) Optimum
* distribution: fill up one broker till its load reaches optimum level (defined by underload threshold) before pull
* another idle broker in; 2) Even distribution: once all brokers' load are above optimum level, maintain all
* brokers to have even load; 3) Set the underload threshold to small value (like 1) for pure even distribution, and
* high value (like 80) for pure optimum distribution;
*
* Strategy to select broker: 1) The first choice is the least loaded broker which is underload but not idle; 2) The
* second choice is idle broker (if there is any); 3) Othewise simply select the least loaded broker if it is NOT
* overloaded; 4) If all brokers are overloaded, select the broker with maximum available capacity (considering
* brokers could have different hardware configuration, this usually means to select the broker with more hardware
* resource);
*
* Broker's load level: 1) Load ranking (triggered by LoadReport update) estimate the load level according to the
* resourse usage and namespace bundles already loaded by each broker; 2) When leader broker decide the owner for a
* new namespace bundle, it may take time for the real owner to actually load the bundle and refresh LoadReport,
* leader broker will store the bundle in a list called preAllocatedBundles, and the quota of all
* preAllocatedBundles in preAllocatedQuotas, and re-estimate the broker's load level by putting the
* preAllocatedQuota into calculation; 3) Everything (preAllocatedBundles and preAllocatedQuotas) will get reset in
* load ranking.
*/
private synchronized ResourceUnit findBrokerForPlacement(Multimap<Long, ResourceUnit> candidates, ServiceUnitId serviceUnit) {
long underloadThreshold = this.getLoadBalancerBrokerUnderloadedThresholdPercentage();
long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
ResourceQuota defaultQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
double minLoadPercentage = 101.0;
long maxAvailability = -1;
ResourceUnit idleRU = null;
ResourceUnit maxAvailableRU = null;
ResourceUnit randomRU = null;
ResourceUnit selectedRU = null;
ResourceUnitRanking selectedRanking = null;
String serviceUnitId = serviceUnit.toString();
// If the ranking is expected to be in the range [0,100] (which is the case for LOADBALANCER_STRATEGY_LLS),
// the ranks are bounded. Otherwise (as is the case in LOADBALANCER_STRATEGY_LEAST_MSG, the ranks are simply
// the total message rate which is in the range [0,Infinity) so they are unbounded. The
// "boundedness" affects how two ranks are compared to see which one is better
boolean unboundedRanks = getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LEAST_MSG);
long randomBrokerIndex = (candidates.size() > 0) ? (this.brokerRotationCursor % candidates.size()) : 0;
// find the least loaded & not-idle broker
for (Map.Entry<Long, ResourceUnit> candidateOwner : candidates.entries()) {
ResourceUnit candidate = candidateOwner.getValue();
randomBrokerIndex--;
// skip broker which is not ranked. this should never happen except in unit test
if (!resourceUnitRankings.containsKey(candidate)) {
continue;
}
String resourceUnitId = candidate.getResourceId();
ResourceUnitRanking ranking = resourceUnitRankings.get(candidate);
// check if this ServiceUnit is already loaded
if (ranking.isServiceUnitLoaded(serviceUnitId)) {
ranking.removeLoadedServiceUnit(serviceUnitId, this.getResourceQuota(serviceUnitId));
}
// record a random broker
if (randomBrokerIndex < 0 && randomRU == null) {
randomRU = candidate;
}
// check the available capacity
double loadPercentage = ranking.getEstimatedLoadPercentage();
double availablePercentage = Math.max(0, (100 - loadPercentage) / 100);
long availability = (long) (ranking.estimateMaxCapacity(defaultQuota) * availablePercentage);
if (availability > maxAvailability) {
maxAvailability = availability;
maxAvailableRU = candidate;
}
// check the load percentage
if (ranking.isIdle()) {
if (idleRU == null) {
idleRU = candidate;
}
} else {
if (selectedRU == null) {
selectedRU = candidate;
selectedRanking = ranking;
minLoadPercentage = loadPercentage;
} else {
if ((unboundedRanks ? ranking.compareMessageRateTo(selectedRanking) : ranking.compareTo(selectedRanking)) < 0) {
minLoadPercentage = loadPercentage;
selectedRU = candidate;
selectedRanking = ranking;
}
}
}
}
if ((minLoadPercentage > underloadThreshold && idleRU != null) || selectedRU == null) {
// assigned to idle broker is the least loaded broker already have optimum load (which means NOT
// underloaded), or all brokers are idle
selectedRU = idleRU;
} else if (minLoadPercentage >= 100.0 && randomRU != null && !unboundedRanks) {
// all brokers are full, assign to a random one
selectedRU = randomRU;
} else if (minLoadPercentage > overloadThreshold && !unboundedRanks) {
// assign to the broker with maximum available capacity if all brokers are overloaded
selectedRU = maxAvailableRU;
}
// re-calculate load level for selected broker
if (selectedRU != null) {
this.brokerRotationCursor = (this.brokerRotationCursor + 1) % 1000000;
ResourceUnitRanking ranking = resourceUnitRankings.get(selectedRU);
String loadPercentageDesc = ranking.getEstimatedLoadPercentageString();
log.info("Assign {} to {} with ({}).", serviceUnitId, selectedRU.getResourceId(), loadPercentageDesc);
if (!ranking.isServiceUnitPreAllocated(serviceUnitId)) {
final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(serviceUnitId);
final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(serviceUnitId);
ResourceQuota quota = this.getResourceQuota(serviceUnitId);
// Add preallocated bundle range so incoming bundles from the same namespace are not assigned to the
// same broker.
brokerToNamespaceToBundleRange.computeIfAbsent(selectedRU.getResourceId().replace("http://", ""), k -> new HashMap<>()).computeIfAbsent(namespaceName, k -> new HashSet<>()).add(bundleRange);
ranking.addPreAllocatedServiceUnit(serviceUnitId, quota);
resourceUnitRankings.put(selectedRU, ranking);
}
}
return selectedRU;
}
Aggregations