Examples with ResourceUnit com.yahoo.pulsar.broker.loadbalance.ResourceUnit used on opensource projects

Example 1 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method updateRealtimeResourceQuota.

private synchronized void updateRealtimeResourceQuota() {
    long memObjectGroupSize = 500;
    if (!currentLoadReports.isEmpty()) {
        long totalBundles = 0;
        long totalMemGroups = 0;
        double totalMsgRateIn = 0.0;
        double totalMsgRateOut = 0.0;
        double totalMsgRate = 0.0;
        double totalCpuUsage = 0.0;
        double totalMemoryUsage = 0.0;
        double totalBandwidthIn = 0.0;
        double totalBandwidthOut = 0.0;
        long loadReportTimestamp = -1;
        // update resource factors
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            LoadReport loadReport = entry.getValue();
            if (loadReport.getTimestamp() > loadReportTimestamp) {
                loadReportTimestamp = loadReport.getTimestamp();
            }
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                totalBundles++;
                NamespaceBundleStats stats = statsEntry.getValue();
                totalMemGroups += (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                totalBandwidthIn += stats.msgThroughputIn;
                totalBandwidthOut += stats.msgThroughputOut;
            }
            SystemResourceUsage resUsage = loadReport.getSystemResourceUsage();
            totalMsgRateIn += loadReport.getMsgRateIn();
            totalMsgRateOut += loadReport.getMsgRateOut();
            totalCpuUsage = totalCpuUsage + resUsage.getCpu().usage;
            totalMemoryUsage = totalMemoryUsage + resUsage.getMemory().usage;
        }
        totalMsgRate = totalMsgRateIn + totalMsgRateOut;
        long timePast = loadReportTimestamp - this.lastResourceQuotaUpdateTimestamp;
        this.lastResourceQuotaUpdateTimestamp = loadReportTimestamp;
        if (totalMsgRate > 1000 && totalMemGroups > 30) {
            this.realtimeCpuLoadFactor = timeSmoothValue(this.realtimeCpuLoadFactor, totalCpuUsage / totalMsgRate, RESOURCE_QUOTA_MIN_CPU_FACTOR, RESOURCE_QUOTA_MAX_CPU_FACTOR, timePast);
            this.realtimeMemoryLoadFactor = timeSmoothValue(this.realtimeMemoryLoadFactor, totalMemoryUsage / totalMemGroups, RESOURCE_QUOTA_MIN_MEM_FACTOR, RESOURCE_QUOTA_MAX_MEM_FACTOR, timePast);
        }
        // calculate average bundle
        if (totalBundles > 30 && this.realtimeAvgResourceQuota.getDynamic()) {
            ResourceQuota oldQuota = this.realtimeAvgResourceQuota;
            ResourceQuota newQuota = timeSmoothQuota(oldQuota, totalMsgRateIn / totalBundles, totalMsgRateOut / totalBundles, totalBandwidthIn / totalBundles, totalBandwidthOut / totalBundles, totalMemoryUsage / totalBundles, timePast);
            this.realtimeAvgResourceQuota = newQuota;
        }
        // update realtime quota for each bundle
        Map<String, ResourceQuota> newQuotas = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                String bundle = statsEntry.getKey();
                NamespaceBundleStats stats = statsEntry.getValue();
                long memGroupCount = (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                double newMemoryQuota = memGroupCount * this.realtimeMemoryLoadFactor;
                ResourceQuota oldQuota = getResourceQuota(bundle);
                ResourceQuota newQuota = timeSmoothQuota(oldQuota, stats.msgRateIn, stats.msgRateOut, stats.msgThroughputIn, stats.msgThroughputOut, newMemoryQuota, timePast);
                newQuotas.put(bundle, newQuota);
            }
        }
        this.realtimeResourceQuotas.set(newQuotas);
    }
}

Example 2 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method doLoadRanking.

/**
     * Rank brokers by available capacity, or load percentage, based on placement strategy:
     *
     * - Available capacity for weighted random selection (weightedRandomSelection): ranks ResourceUnits units based on
     * estimation of their capacity which is basically how many bundles each ResourceUnit is able can handle with its
     * available resources (CPU, memory, network, etc);
     *
     * - Load percentage for least loaded server (leastLoadedServer): ranks ResourceUnits units based on estimation of
     * their load percentage which is basically how many percent of resource is allocated which is
     * max(resource_actually_used, resource_quota)
     *
     * If we fail to collect the Load Reports OR fail to process them for the first time, it means the leader does not
     * have enough information to make a decision so we set it to ready when we collect and process the load reports
     * successfully the first time.
     */
private synchronized void doLoadRanking() {
    ResourceUnitRanking.setCpuUsageByMsgRate(this.realtimeCpuLoadFactor);
    String hostname = pulsar.getAdvertisedAddress();
    String strategy = this.getLoadBalancerPlacementStrategy();
    log.info("doLoadRanking - load balancing strategy: {}", strategy);
    if (!currentLoadReports.isEmpty()) {
        synchronized (resourceUnitRankings) {
            Map<Long, Set<ResourceUnit>> newSortedRankings = Maps.newTreeMap();
            Map<ResourceUnit, ResourceUnitRanking> newResourceUnitRankings = new HashMap<>();
            for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
                ResourceUnit resourceUnit = entry.getKey();
                LoadReport loadReport = entry.getValue();
                // calculate rankings
                Set<String> loadedBundles = loadReport.getBundles();
                Set<String> preAllocatedBundles = null;
                if (resourceUnitRankings.containsKey(resourceUnit)) {
                    preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles();
                    preAllocatedBundles.removeAll(loadedBundles);
                } else {
                    preAllocatedBundles = new HashSet<>();
                }
                ResourceQuota allocatedQuota = getTotalAllocatedQuota(loadedBundles);
                ResourceQuota preAllocatedQuota = getTotalAllocatedQuota(preAllocatedBundles);
                ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), loadedBundles, allocatedQuota, preAllocatedBundles, preAllocatedQuota);
                newResourceUnitRankings.put(resourceUnit, ranking);
                // generated sorted ranking
                double loadPercentage = ranking.getEstimatedLoadPercentage();
                long maxCapacity = ranking.estimateMaxCapacity(pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota());
                long finalRank = 0;
                if (strategy.equals(LOADBALANCER_STRATEGY_LLS)) {
                    finalRank = (long) loadPercentage;
                } else {
                    double idleRatio = (100 - loadPercentage) / 100;
                    finalRank = (long) (maxCapacity * idleRatio * idleRatio);
                }
                if (!newSortedRankings.containsKey(finalRank)) {
                    newSortedRankings.put(finalRank, new HashSet<ResourceUnit>());
                }
                newSortedRankings.get(finalRank).add(entry.getKey());
                if (log.isDebugEnabled()) {
                    log.debug("Added Resource Unit [{}] with Rank [{}]", entry.getKey().getResourceId(), finalRank);
                }
                // update metrics
                if (resourceUnit.getResourceId().contains(hostname)) {
                    updateLoadBalancingMetrics(hostname, finalRank, ranking);
                }
            }
            this.sortedRankings.set(newSortedRankings);
            this.resourceUnitRankings = newResourceUnitRankings;
        }
    } else {
        log.info("Leader broker[{}] No ResourceUnits to rank this run, Using Old Ranking", pulsar.getWebServiceAddress());
    }
}

Example 3 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method isBrokerAvailableForRebalancing.

// todo: changeme: this can be optimized, we don't have to iterate through everytime
private boolean isBrokerAvailableForRebalancing(String bundleName, long maxLoadLevel) {
    NamespaceName namespaceName = new NamespaceName(getNamespaceNameFromBundleName(bundleName));
    Map<Long, Set<ResourceUnit>> availableBrokers = sortedRankings.get();
    // this does not have "http://" in front, hacky but no time to pretty up
    Multimap<Long, ResourceUnit> brokers = getFinalCandidates(namespaceName, availableBrokers);
    for (Object broker : brokers.values()) {
        ResourceUnit underloadedRU = (ResourceUnit) broker;
        LoadReport currentLoadReport = currentLoadReports.get(underloadedRU);
        if (isBelowLoadLevel(currentLoadReport.getSystemResourceUsage(), maxLoadLevel)) {
            return true;
        }
    }
    return false;
}

Example 4 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method getAvailableBrokers.

private Map<Long, Set<ResourceUnit>> getAvailableBrokers(ServiceUnitId serviceUnitId) throws Exception {
    Map<Long, Set<ResourceUnit>> availableBrokers = sortedRankings.get();
    if (availableBrokers.isEmpty()) {
        // Create a map with all available brokers with no load information
        Set<String> activeBrokers = availableActiveBrokers.get(LOADBALANCE_BROKERS_ROOT);
        List<String> brokersToShuffle = new ArrayList<>(activeBrokers);
        Collections.shuffle(brokersToShuffle);
        activeBrokers = new HashSet<>(brokersToShuffle);
        availableBrokers = Maps.newTreeMap();
        for (String broker : activeBrokers) {
            ResourceUnit resourceUnit = new SimpleResourceUnit(String.format("http://%s", broker), new PulsarResourceDescription());
            availableBrokers.computeIfAbsent(0L, key -> Sets.newTreeSet()).add(resourceUnit);
        }
        log.info("Choosing at random from broker list: [{}]", availableBrokers.values());
    }
    return availableBrokers;
}

Example 5 with ResourceUnit

use of com.yahoo.pulsar.broker.loadbalance.ResourceUnit in project pulsar by yahoo.

the class SimpleLoadManagerImpl method updateRanking.

private void updateRanking() {
    try {
        synchronized (currentLoadReports) {
            currentLoadReports.clear();
            Set<String> activeBrokers = availableActiveBrokers.get();
            for (String broker : activeBrokers) {
                try {
                    String key = String.format("%s/%s", LOADBALANCE_BROKERS_ROOT, broker);
                    LoadReport lr = loadReportCacheZk.get(key).orElseThrow(() -> new KeeperException.NoNodeException());
                    ResourceUnit ru = new SimpleResourceUnit(String.format("http://%s", lr.getName()), fromLoadReport(lr));
                    this.currentLoadReports.put(ru, lr);
                } catch (Exception e) {
                    log.warn("Error reading load report from Cache for broker - [{}], [{}]", broker, e);
                }
            }
            updateRealtimeResourceQuota();
            doLoadRanking();
        }
    } catch (Exception e) {
        log.warn("Error reading active brokers list from zookeeper while re-ranking load reports [{}]", e);
    }
}