Examples with ResourceQuota com.yahoo.pulsar.common.policies.data.ResourceQuota used on opensource projects

Example 6 with ResourceQuota

use of com.yahoo.pulsar.common.policies.data.ResourceQuota in project pulsar by yahoo.

the class LoadBalancerTest method testDestinationAssignmentWithExistingBundles.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - already has some bundles
     * assigned Check the distribution of new destinations is roughly consistent (with <10% variation) with the ranking
     */
@Test
public void testDestinationAssignmentWithExistingBundles() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        ResourceQuota defaultQuota = new ResourceQuota();
        defaultQuota.setMsgRateIn(20);
        defaultQuota.setMsgRateOut(60);
        defaultQuota.setBandwidthIn(20000);
        defaultQuota.setBandwidthOut(60000);
        defaultQuota.setMemory(87);
        pulsarServices[i].getLocalZkCacheService().getResourceQuotaCache().setDefaultQuota(defaultQuota);
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(0, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < (i + 1) * 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = objectMapper.writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // print ranking
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
    }
    // check owner of new destiations and verify that the distribution is roughly
    // consistent (variation < 10%) with the broker capacity:
    int totalNamespaces = 250;
    int[] expectedAssignments = new int[] { 17, 34, 51, 68, 85 };
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        DestinationName fqdn = DestinationName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(fqdn));
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    double expectedMaxVariation = 10.0;
    for (int i = 0; i < BROKER_COUNT; i++) {
        long actualValue = 0;
        String resourceId = "http://" + lookupAddresses[i];
        if (namespaceOwner.containsKey(resourceId)) {
            actualValue = namespaceOwner.get(resourceId);
        }
        long expectedValue = expectedAssignments[i];
        double variation = Math.abs(actualValue - expectedValue) * 100.0 / expectedValue;
        log.info("Destination assignment - {}, actual: {}, expected baseline: {}, variation: {}/%", lookupAddresses[i], actualValue, expectedValue, String.format("%.2f", variation));
        assertTrue(variation < expectedMaxVariation);
    }
}

Example 7 with ResourceQuota

use of com.yahoo.pulsar.common.policies.data.ResourceQuota in project pulsar by yahoo.

the class LoadBalancerTest method testDynamicNamespaceBundleQuota.

/*
     * Test broker dynamically calculating resource quota for each connected namespace bundle.
     */
@Test
public void testDynamicNamespaceBundleQuota() throws Exception {
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < BROKER_COUNT; i++) {
        ResourceQuota defaultQuota = new ResourceQuota();
        defaultQuota.setMsgRateIn(20);
        defaultQuota.setMsgRateOut(60);
        defaultQuota.setBandwidthIn(20000);
        defaultQuota.setBandwidthOut(60000);
        defaultQuota.setMemory(75);
        pulsarServices[i].getLocalZkCacheService().getResourceQuotaCache().setDefaultQuota(defaultQuota);
    }
    // publish the initial load reports and wait for quotas be updated
    writeLoadReportsForDynamicQuota(startTime);
    Thread.sleep(5000);
    // publish test report for 15 minutes later and wait for quotas be updated
    writeLoadReportsForDynamicQuota(startTime + SimpleLoadManagerImpl.RESOURCE_QUOTA_GO_UP_TIMEWINDOW);
    Thread.sleep(5000);
    // print & verify resource quotas
    for (int i = 0; i < BROKER_COUNT; i++) {
        Map<String, ResourceQuota> quotas = getRealtimeResourceQuota(pulsarServices[i]).get();
        printResourceQuotas(quotas);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-0-0/0x00000000_0xffffffff"), 19.0, 58.0, 19791.0, 58958.0, 74.0);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-2-2/0x00000000_0xffffffff"), 20.0, 60.0, 20000.0, 60000.0, 100.0);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-4-4/0x00000000_0xffffffff"), 40.0, 120.0, 40000.0, 120000.0, 150.0);
    }
    // publish test report for 24 hours later and wait for quotas be updated
    writeLoadReportsForDynamicQuota(startTime + SimpleLoadManagerImpl.RESOURCE_QUOTA_GO_DOWN_TIMEWINDOW);
    Thread.sleep(5000);
    // print & verify resource quotas
    for (int i = 0; i < BROKER_COUNT; i++) {
        Map<String, ResourceQuota> quotas = getRealtimeResourceQuota(pulsarServices[i]).get();
        printResourceQuotas(quotas);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-0-0/0x00000000_0xffffffff"), 5.0, 6.0, 10203.0, 11019.0, 50.0);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-2-2/0x00000000_0xffffffff"), 20.0, 60.0, 20000.0, 60000.0, 100.0);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-4-4/0x00000000_0xffffffff"), 40.0, 120.0, 40000.0, 120000.0, 150.0);
    }
}

Example 8 with ResourceQuota

use of com.yahoo.pulsar.common.policies.data.ResourceQuota in project pulsar by yahoo.

the class SimpleLoadManagerImpl method updateRealtimeResourceQuota.

private synchronized void updateRealtimeResourceQuota() {
    long memObjectGroupSize = 500;
    if (!currentLoadReports.isEmpty()) {
        long totalBundles = 0;
        long totalMemGroups = 0;
        double totalMsgRateIn = 0.0;
        double totalMsgRateOut = 0.0;
        double totalMsgRate = 0.0;
        double totalCpuUsage = 0.0;
        double totalMemoryUsage = 0.0;
        double totalBandwidthIn = 0.0;
        double totalBandwidthOut = 0.0;
        long loadReportTimestamp = -1;
        // update resource factors
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            LoadReport loadReport = entry.getValue();
            if (loadReport.getTimestamp() > loadReportTimestamp) {
                loadReportTimestamp = loadReport.getTimestamp();
            }
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                totalBundles++;
                NamespaceBundleStats stats = statsEntry.getValue();
                totalMemGroups += (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                totalBandwidthIn += stats.msgThroughputIn;
                totalBandwidthOut += stats.msgThroughputOut;
            }
            SystemResourceUsage resUsage = loadReport.getSystemResourceUsage();
            totalMsgRateIn += loadReport.getMsgRateIn();
            totalMsgRateOut += loadReport.getMsgRateOut();
            totalCpuUsage = totalCpuUsage + resUsage.getCpu().usage;
            totalMemoryUsage = totalMemoryUsage + resUsage.getMemory().usage;
        }
        totalMsgRate = totalMsgRateIn + totalMsgRateOut;
        long timePast = loadReportTimestamp - this.lastResourceQuotaUpdateTimestamp;
        this.lastResourceQuotaUpdateTimestamp = loadReportTimestamp;
        if (totalMsgRate > 1000 && totalMemGroups > 30) {
            this.realtimeCpuLoadFactor = timeSmoothValue(this.realtimeCpuLoadFactor, totalCpuUsage / totalMsgRate, RESOURCE_QUOTA_MIN_CPU_FACTOR, RESOURCE_QUOTA_MAX_CPU_FACTOR, timePast);
            this.realtimeMemoryLoadFactor = timeSmoothValue(this.realtimeMemoryLoadFactor, totalMemoryUsage / totalMemGroups, RESOURCE_QUOTA_MIN_MEM_FACTOR, RESOURCE_QUOTA_MAX_MEM_FACTOR, timePast);
        }
        // calculate average bundle
        if (totalBundles > 30 && this.realtimeAvgResourceQuota.getDynamic()) {
            ResourceQuota oldQuota = this.realtimeAvgResourceQuota;
            ResourceQuota newQuota = timeSmoothQuota(oldQuota, totalMsgRateIn / totalBundles, totalMsgRateOut / totalBundles, totalBandwidthIn / totalBundles, totalBandwidthOut / totalBundles, totalMemoryUsage / totalBundles, timePast);
            this.realtimeAvgResourceQuota = newQuota;
        }
        // update realtime quota for each bundle
        Map<String, ResourceQuota> newQuotas = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                String bundle = statsEntry.getKey();
                NamespaceBundleStats stats = statsEntry.getValue();
                long memGroupCount = (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                double newMemoryQuota = memGroupCount * this.realtimeMemoryLoadFactor;
                ResourceQuota oldQuota = getResourceQuota(bundle);
                ResourceQuota newQuota = timeSmoothQuota(oldQuota, stats.msgRateIn, stats.msgRateOut, stats.msgThroughputIn, stats.msgThroughputOut, newMemoryQuota, timePast);
                newQuotas.put(bundle, newQuota);
            }
        }
        this.realtimeResourceQuotas.set(newQuotas);
    }
}

Example 9 with ResourceQuota

use of com.yahoo.pulsar.common.policies.data.ResourceQuota in project pulsar by yahoo.

the class SimpleLoadManagerImpl method doLoadRanking.

/**
     * Rank brokers by available capacity, or load percentage, based on placement strategy:
     *
     * - Available capacity for weighted random selection (weightedRandomSelection): ranks ResourceUnits units based on
     * estimation of their capacity which is basically how many bundles each ResourceUnit is able can handle with its
     * available resources (CPU, memory, network, etc);
     *
     * - Load percentage for least loaded server (leastLoadedServer): ranks ResourceUnits units based on estimation of
     * their load percentage which is basically how many percent of resource is allocated which is
     * max(resource_actually_used, resource_quota)
     *
     * If we fail to collect the Load Reports OR fail to process them for the first time, it means the leader does not
     * have enough information to make a decision so we set it to ready when we collect and process the load reports
     * successfully the first time.
     */
private synchronized void doLoadRanking() {
    ResourceUnitRanking.setCpuUsageByMsgRate(this.realtimeCpuLoadFactor);
    String hostname = pulsar.getAdvertisedAddress();
    String strategy = this.getLoadBalancerPlacementStrategy();
    log.info("doLoadRanking - load balancing strategy: {}", strategy);
    if (!currentLoadReports.isEmpty()) {
        synchronized (resourceUnitRankings) {
            Map<Long, Set<ResourceUnit>> newSortedRankings = Maps.newTreeMap();
            Map<ResourceUnit, ResourceUnitRanking> newResourceUnitRankings = new HashMap<>();
            for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
                ResourceUnit resourceUnit = entry.getKey();
                LoadReport loadReport = entry.getValue();
                // calculate rankings
                Set<String> loadedBundles = loadReport.getBundles();
                Set<String> preAllocatedBundles = null;
                if (resourceUnitRankings.containsKey(resourceUnit)) {
                    preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles();
                    preAllocatedBundles.removeAll(loadedBundles);
                } else {
                    preAllocatedBundles = new HashSet<>();
                }
                ResourceQuota allocatedQuota = getTotalAllocatedQuota(loadedBundles);
                ResourceQuota preAllocatedQuota = getTotalAllocatedQuota(preAllocatedBundles);
                ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), loadedBundles, allocatedQuota, preAllocatedBundles, preAllocatedQuota);
                newResourceUnitRankings.put(resourceUnit, ranking);
                // generated sorted ranking
                double loadPercentage = ranking.getEstimatedLoadPercentage();
                long maxCapacity = ranking.estimateMaxCapacity(pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota());
                long finalRank = 0;
                if (strategy.equals(LOADBALANCER_STRATEGY_LLS)) {
                    finalRank = (long) loadPercentage;
                } else {
                    double idleRatio = (100 - loadPercentage) / 100;
                    finalRank = (long) (maxCapacity * idleRatio * idleRatio);
                }
                if (!newSortedRankings.containsKey(finalRank)) {
                    newSortedRankings.put(finalRank, new HashSet<ResourceUnit>());
                }
                newSortedRankings.get(finalRank).add(entry.getKey());
                if (log.isDebugEnabled()) {
                    log.debug("Added Resource Unit [{}] with Rank [{}]", entry.getKey().getResourceId(), finalRank);
                }
                // update metrics
                if (resourceUnit.getResourceId().contains(hostname)) {
                    updateLoadBalancingMetrics(hostname, finalRank, ranking);
                }
            }
            this.sortedRankings.set(newSortedRankings);
            this.resourceUnitRankings = newResourceUnitRankings;
        }
    } else {
        log.info("Leader broker[{}] No ResourceUnits to rank this run, Using Old Ranking", pulsar.getWebServiceAddress());
    }
}

Example 10 with ResourceQuota

use of com.yahoo.pulsar.common.policies.data.ResourceQuota in project pulsar by yahoo.

the class SimpleLoadManagerImpl method getResourceQuota.

private ResourceQuota getResourceQuota(String bundle) {
    Map<String, ResourceQuota> quotas = this.realtimeResourceQuotas.get();
    if (!quotas.containsKey(bundle)) {
        ResourceQuota quota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getQuota(bundle);
        quotas.put(bundle, quota);
        return quota;
    } else {
        return quotas.get(bundle);
    }
}