Examples with ResourceQuota org.apache.pulsar.common.policies.data.ResourceQuota used on opensource projects

Example 1 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method getBundleDataOrDefault.

// Attempt to local the data for the given bundle in ZooKeeper.
// If it cannot be found, return the default bundle data.
private BundleData getBundleDataOrDefault(final String bundle) {
    BundleData bundleData = null;
    try {
        final String bundleZPath = getBundleDataZooKeeperPath(bundle);
        final String quotaZPath = String.format("%s/%s", RESOURCE_QUOTA_ZPATH, bundle);
        if (zkClient.exists(bundleZPath, null) != null) {
            bundleData = readJson(zkClient.getData(bundleZPath, null, null), BundleData.class);
        } else if (zkClient.exists(quotaZPath, null) != null) {
            final ResourceQuota quota = readJson(zkClient.getData(quotaZPath, null, null), ResourceQuota.class);
            bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES);
            // Initialize from existing resource quotas if new API ZNodes do not exist.
            final TimeAverageMessageData shortTermData = bundleData.getShortTermData();
            final TimeAverageMessageData longTermData = bundleData.getLongTermData();
            shortTermData.setMsgRateIn(quota.getMsgRateIn());
            shortTermData.setMsgRateOut(quota.getMsgRateOut());
            shortTermData.setMsgThroughputIn(quota.getBandwidthIn());
            shortTermData.setMsgThroughputOut(quota.getBandwidthOut());
            longTermData.setMsgRateIn(quota.getMsgRateIn());
            longTermData.setMsgRateOut(quota.getMsgRateOut());
            longTermData.setMsgThroughputIn(quota.getBandwidthIn());
            longTermData.setMsgThroughputOut(quota.getBandwidthOut());
            // Assume ample history.
            shortTermData.setNumSamples(NUM_SHORT_SAMPLES);
            longTermData.setNumSamples(NUM_LONG_SAMPLES);
        }
    } catch (Exception e) {
        log.warn("Error when trying to find bundle {} on zookeeper: {}", bundle, e);
    }
    if (bundleData == null) {
        bundleData = new BundleData(NUM_SHORT_SAMPLES, NUM_LONG_SAMPLES, defaultStats);
    }
    return bundleData;
}

Example 2 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method timeSmoothQuota.

private ResourceQuota timeSmoothQuota(ResourceQuota oldQuota, double msgRateIn, double msgRateOut, double bandwidthIn, double bandwidthOut, double memory, long timePast) {
    if (oldQuota.getDynamic()) {
        ResourceQuota newQuota = new ResourceQuota();
        newQuota.setMsgRateIn(timeSmoothValue(oldQuota.getMsgRateIn(), msgRateIn, RESOURCE_QUOTA_MIN_MSGRATE_IN, RESOURCE_QUOTA_MAX_MSGRATE_IN, timePast));
        newQuota.setMsgRateOut(timeSmoothValue(oldQuota.getMsgRateOut(), msgRateOut, RESOURCE_QUOTA_MIN_MSGRATE_OUT, RESOURCE_QUOTA_MAX_MSGRATE_OUT, timePast));
        newQuota.setBandwidthIn(timeSmoothValue(oldQuota.getBandwidthIn(), bandwidthIn, RESOURCE_QUOTA_MIN_BANDWIDTH_IN, RESOURCE_QUOTA_MAX_BANDWIDTH_IN, timePast));
        newQuota.setBandwidthOut(timeSmoothValue(oldQuota.getBandwidthOut(), bandwidthOut, RESOURCE_QUOTA_MIN_BANDWIDTH_OUT, RESOURCE_QUOTA_MAX_BANDWIDTH_OUT, timePast));
        newQuota.setMemory(timeSmoothValue(oldQuota.getMemory(), memory, RESOURCE_QUOTA_MIN_MEMORY, RESOURCE_QUOTA_MAX_MEMORY, timePast));
        return newQuota;
    } else {
        return oldQuota;
    }
}

Example 3 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method updateRealtimeResourceQuota.

private synchronized void updateRealtimeResourceQuota() {
    long memObjectGroupSize = 500;
    if (!currentLoadReports.isEmpty()) {
        long totalBundles = 0;
        long totalMemGroups = 0;
        double totalMsgRateIn = 0.0;
        double totalMsgRateOut = 0.0;
        double totalMsgRate = 0.0;
        double totalCpuUsage = 0.0;
        double totalMemoryUsage = 0.0;
        double totalBandwidthIn = 0.0;
        double totalBandwidthOut = 0.0;
        long loadReportTimestamp = -1;
        // update resource factors
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            LoadReport loadReport = entry.getValue();
            if (loadReport.getTimestamp() > loadReportTimestamp) {
                loadReportTimestamp = loadReport.getTimestamp();
            }
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                totalBundles++;
                NamespaceBundleStats stats = statsEntry.getValue();
                totalMemGroups += (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                totalBandwidthIn += stats.msgThroughputIn;
                totalBandwidthOut += stats.msgThroughputOut;
            }
            SystemResourceUsage resUsage = loadReport.getSystemResourceUsage();
            totalMsgRateIn += loadReport.getMsgRateIn();
            totalMsgRateOut += loadReport.getMsgRateOut();
            totalCpuUsage = totalCpuUsage + resUsage.getCpu().usage;
            totalMemoryUsage = totalMemoryUsage + resUsage.getMemory().usage;
        }
        totalMsgRate = totalMsgRateIn + totalMsgRateOut;
        long timePast = loadReportTimestamp - this.lastResourceQuotaUpdateTimestamp;
        this.lastResourceQuotaUpdateTimestamp = loadReportTimestamp;
        if (totalMsgRate > 1000 && totalMemGroups > 30) {
            this.realtimeCpuLoadFactor = timeSmoothValue(this.realtimeCpuLoadFactor, totalCpuUsage / totalMsgRate, RESOURCE_QUOTA_MIN_CPU_FACTOR, RESOURCE_QUOTA_MAX_CPU_FACTOR, timePast);
            this.realtimeMemoryLoadFactor = timeSmoothValue(this.realtimeMemoryLoadFactor, totalMemoryUsage / totalMemGroups, RESOURCE_QUOTA_MIN_MEM_FACTOR, RESOURCE_QUOTA_MAX_MEM_FACTOR, timePast);
        }
        // calculate average bundle
        if (totalBundles > 30 && this.realtimeAvgResourceQuota.getDynamic()) {
            ResourceQuota oldQuota = this.realtimeAvgResourceQuota;
            ResourceQuota newQuota = timeSmoothQuota(oldQuota, totalMsgRateIn / totalBundles, totalMsgRateOut / totalBundles, totalBandwidthIn / totalBundles, totalBandwidthOut / totalBundles, totalMemoryUsage / totalBundles, timePast);
            this.realtimeAvgResourceQuota = newQuota;
        }
        // update realtime quota for each bundle
        Map<String, ResourceQuota> newQuotas = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                String bundle = statsEntry.getKey();
                NamespaceBundleStats stats = statsEntry.getValue();
                long memGroupCount = (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                double newMemoryQuota = memGroupCount * this.realtimeMemoryLoadFactor;
                ResourceQuota oldQuota = getResourceQuota(bundle);
                ResourceQuota newQuota = timeSmoothQuota(oldQuota, stats.msgRateIn, stats.msgRateOut, stats.msgThroughputIn, stats.msgThroughputOut, newMemoryQuota, timePast);
                newQuotas.put(bundle, newQuota);
            }
        }
        this.realtimeResourceQuotas.set(newQuotas);
    }
}

Example 4 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method findBrokerForPlacement.

/**
 * Assign owner for specified ServiceUnit from the given candidates, following the the principles: 1) Optimum
 * distribution: fill up one broker till its load reaches optimum level (defined by underload threshold) before pull
 * another idle broker in; 2) Even distribution: once all brokers' load are above optimum level, maintain all
 * brokers to have even load; 3) Set the underload threshold to small value (like 1) for pure even distribution, and
 * high value (like 80) for pure optimum distribution;
 *
 * Strategy to select broker: 1) The first choice is the least loaded broker which is underload but not idle; 2) The
 * second choice is idle broker (if there is any); 3) Othewise simply select the least loaded broker if it is NOT
 * overloaded; 4) If all brokers are overloaded, select the broker with maximum available capacity (considering
 * brokers could have different hardware configuration, this usually means to select the broker with more hardware
 * resource);
 *
 * Broker's load level: 1) Load ranking (triggered by LoadReport update) estimate the load level according to the
 * resourse usage and namespace bundles already loaded by each broker; 2) When leader broker decide the owner for a
 * new namespace bundle, it may take time for the real owner to actually load the bundle and refresh LoadReport,
 * leader broker will store the bundle in a list called preAllocatedBundles, and the quota of all
 * preAllocatedBundles in preAllocatedQuotas, and re-estimate the broker's load level by putting the
 * preAllocatedQuota into calculation; 3) Everything (preAllocatedBundles and preAllocatedQuotas) will get reset in
 * load ranking.
 */
private synchronized ResourceUnit findBrokerForPlacement(Multimap<Long, ResourceUnit> candidates, ServiceUnitId serviceUnit) {
    long underloadThreshold = this.getLoadBalancerBrokerUnderloadedThresholdPercentage();
    long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
    ResourceQuota defaultQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
    double minLoadPercentage = 101.0;
    long maxAvailability = -1;
    ResourceUnit idleRU = null;
    ResourceUnit maxAvailableRU = null;
    ResourceUnit randomRU = null;
    ResourceUnit selectedRU = null;
    ResourceUnitRanking selectedRanking = null;
    String serviceUnitId = serviceUnit.toString();
    // If the ranking is expected to be in the range [0,100] (which is the case for LOADBALANCER_STRATEGY_LLS),
    // the ranks are bounded. Otherwise (as is the case in LOADBALANCER_STRATEGY_LEAST_MSG, the ranks are simply
    // the total message rate which is in the range [0,Infinity) so they are unbounded. The
    // "boundedness" affects how two ranks are compared to see which one is better
    boolean unboundedRanks = getLoadBalancerPlacementStrategy().equals(LOADBALANCER_STRATEGY_LEAST_MSG);
    long randomBrokerIndex = (candidates.size() > 0) ? (this.brokerRotationCursor % candidates.size()) : 0;
    // find the least loaded & not-idle broker
    for (Map.Entry<Long, ResourceUnit> candidateOwner : candidates.entries()) {
        ResourceUnit candidate = candidateOwner.getValue();
        randomBrokerIndex--;
        // skip broker which is not ranked. this should never happen except in unit test
        if (!resourceUnitRankings.containsKey(candidate)) {
            continue;
        }
        String resourceUnitId = candidate.getResourceId();
        ResourceUnitRanking ranking = resourceUnitRankings.get(candidate);
        // check if this ServiceUnit is already loaded
        if (ranking.isServiceUnitLoaded(serviceUnitId)) {
            ranking.removeLoadedServiceUnit(serviceUnitId, this.getResourceQuota(serviceUnitId));
        }
        // record a random broker
        if (randomBrokerIndex < 0 && randomRU == null) {
            randomRU = candidate;
        }
        // check the available capacity
        double loadPercentage = ranking.getEstimatedLoadPercentage();
        double availablePercentage = Math.max(0, (100 - loadPercentage) / 100);
        long availability = (long) (ranking.estimateMaxCapacity(defaultQuota) * availablePercentage);
        if (availability > maxAvailability) {
            maxAvailability = availability;
            maxAvailableRU = candidate;
        }
        // check the load percentage
        if (ranking.isIdle()) {
            if (idleRU == null) {
                idleRU = candidate;
            }
        } else {
            if (selectedRU == null) {
                selectedRU = candidate;
                selectedRanking = ranking;
                minLoadPercentage = loadPercentage;
            } else {
                if ((unboundedRanks ? ranking.compareMessageRateTo(selectedRanking) : ranking.compareTo(selectedRanking)) < 0) {
                    minLoadPercentage = loadPercentage;
                    selectedRU = candidate;
                    selectedRanking = ranking;
                }
            }
        }
    }
    if ((minLoadPercentage > underloadThreshold && idleRU != null) || selectedRU == null) {
        // assigned to idle broker is the least loaded broker already have optimum load (which means NOT
        // underloaded), or all brokers are idle
        selectedRU = idleRU;
    } else if (minLoadPercentage >= 100.0 && randomRU != null && !unboundedRanks) {
        // all brokers are full, assign to a random one
        selectedRU = randomRU;
    } else if (minLoadPercentage > overloadThreshold && !unboundedRanks) {
        // assign to the broker with maximum available capacity if all brokers are overloaded
        selectedRU = maxAvailableRU;
    }
    // re-calculate load level for selected broker
    if (selectedRU != null) {
        this.brokerRotationCursor = (this.brokerRotationCursor + 1) % 1000000;
        ResourceUnitRanking ranking = resourceUnitRankings.get(selectedRU);
        String loadPercentageDesc = ranking.getEstimatedLoadPercentageString();
        log.info("Assign {} to {} with ({}).", serviceUnitId, selectedRU.getResourceId(), loadPercentageDesc);
        if (!ranking.isServiceUnitPreAllocated(serviceUnitId)) {
            final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(serviceUnitId);
            final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(serviceUnitId);
            ResourceQuota quota = this.getResourceQuota(serviceUnitId);
            // Add preallocated bundle range so incoming bundles from the same namespace are not assigned to the
            // same broker.
            brokerToNamespaceToBundleRange.computeIfAbsent(selectedRU.getResourceId().replace("http://", ""), k -> new HashMap<>()).computeIfAbsent(namespaceName, k -> new HashSet<>()).add(bundleRange);
            ranking.addPreAllocatedServiceUnit(serviceUnitId, quota);
            resourceUnitRankings.put(selectedRU, ranking);
        }
    }
    return selectedRU;
}

Example 5 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method getTotalAllocatedQuota.

/**
 * Get the sum of allocated resource for the list of namespace bundles
 */
private ResourceQuota getTotalAllocatedQuota(Set<String> bundles) {
    ResourceQuota totalQuota = new ResourceQuota();
    for (String bundle : bundles) {
        ResourceQuota quota = this.getResourceQuota(bundle);
        totalQuota.add(quota);
    }
    return totalQuota;
}