Examples with ResourceQuota org.apache.pulsar.common.policies.data.ResourceQuota used on opensource projects

Example 16 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class ResourceQuotaTest method testResourceQuotaEqual.

@Test
public void testResourceQuotaEqual() {
    ResourceQuota quota1 = new ResourceQuota();
    quota1.setMsgRateIn(10);
    quota1.setMsgRateOut(20);
    quota1.setBandwidthIn(10000);
    quota1.setBandwidthOut(20000);
    quota1.setMemory(100);
    quota1.setDynamic(false);
    ResourceQuota quota2 = new ResourceQuota();
    Assert.assertNotEquals(quota1, quota2);
    quota2.setMsgRateIn(10);
    Assert.assertNotEquals(quota1, quota2);
    quota2.setMsgRateOut(20);
    Assert.assertNotEquals(quota1, quota2);
    quota2.setBandwidthIn(10000);
    Assert.assertNotEquals(quota1, quota2);
    quota2.setBandwidthOut(20000);
    Assert.assertNotEquals(quota1, quota2);
    quota2.setMemory(100);
    Assert.assertNotEquals(quota1, quota2);
    quota2.setDynamic(false);
    Assert.assertEquals(quota1, quota2);
}

Example 17 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class ResourceQuotaCache method initZK.

/**
 * Initialize the resource quota root node in ZooKeeper.
 */
public void initZK() throws PulsarServerException {
    String zpath = ResourceQuotaCache.path(null);
    ResourceQuota quota = this.readQuotaFromZnode(zpath);
    if (!quota.isValid()) {
        quota = ResourceQuotaCache.getInitialQuotaValue();
        try {
            this.saveQuotaToZnode(zpath, quota);
        } catch (Exception e) {
            throw new PulsarServerException(e);
        }
    }
}

Example 18 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class LoadSimulationController method handleSimulate.

// Handle the command line arguments associated with the simulate command.
private void handleSimulate(final ShellArguments arguments) throws Exception {
    final List<String> commandArguments = arguments.commandArguments;
    checkAppArgs(commandArguments.size() - 1, 1);
    final ZooKeeper zkClient = new ZooKeeper(commandArguments.get(1), 5000, null);
    // Make a map for each thread to speed up the ZooKeeper writing process.
    final Map<String, ResourceQuota>[] threadLocalMaps = new Map[clients.length];
    for (int i = 0; i < clients.length; ++i) {
        threadLocalMaps[i] = new HashMap<>();
    }
    getResourceQuotas(QUOTA_ROOT, zkClient, threadLocalMaps);
    final List<Future> futures = new ArrayList<>(clients.length);
    int i = 0;
    log.info("Simulating...");
    for (final Map<String, ResourceQuota> bundleToQuota : threadLocalMaps) {
        final int j = i;
        futures.add(threadPool.submit(() -> {
            for (final Map.Entry<String, ResourceQuota> entry : bundleToQuota.entrySet()) {
                final String bundle = entry.getKey();
                final String newAPIPath = bundle.replace(QUOTA_ROOT, BUNDLE_DATA_ROOT);
                final ResourceQuota quota = entry.getValue();
                final int tenantStart = QUOTA_ROOT.length() + 1;
                final String topic = String.format("persistent://%s/t", bundle.substring(tenantStart));
                final BundleData bundleData = initializeBundleData(quota, arguments);
                // Put the bundle data in the new ZooKeeper.
                try {
                    ZkUtils.createFullPathOptimistic(zkClient, newAPIPath, bundleData.getJsonBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
                } catch (KeeperException.NodeExistsException e) {
                    try {
                        zkClient.setData(newAPIPath, bundleData.getJsonBytes(), -1);
                    } catch (Exception ex) {
                        throw new RuntimeException(ex);
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
                try {
                    trade(arguments, topic, j);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        }));
        ++i;
    }
    for (final Future future : futures) {
        future.get();
    }
    zkClient.close();
}

Example 19 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class PulsarAdminToolTest method resourceQuotas.

@Test
void resourceQuotas() throws Exception {
    PulsarAdmin admin = Mockito.mock(PulsarAdmin.class);
    ResourceQuotas mockResourceQuotas = mock(ResourceQuotas.class);
    when(admin.resourceQuotas()).thenReturn(mockResourceQuotas);
    CmdResourceQuotas cmdResourceQuotas = new CmdResourceQuotas(admin);
    ResourceQuota quota = new ResourceQuota();
    quota.setMsgRateIn(10);
    quota.setMsgRateOut(20);
    quota.setBandwidthIn(10000);
    quota.setBandwidthOut(20000);
    quota.setMemory(100);
    quota.setDynamic(false);
    cmdResourceQuotas.run(split("get"));
    verify(mockResourceQuotas).getDefaultResourceQuota();
    cmdResourceQuotas.run(split("set -mi 10 -mo 20 -bi 10000 -bo 20000 -mem 100"));
    verify(mockResourceQuotas).setDefaultResourceQuota(quota);
    // reset mocks
    mockResourceQuotas = mock(ResourceQuotas.class);
    when(admin.resourceQuotas()).thenReturn(mockResourceQuotas);
    cmdResourceQuotas = new CmdResourceQuotas(admin);
    cmdResourceQuotas.run(split("get --namespace myprop/clust/ns1 --bundle 0x80000000_0xffffffff"));
    verify(mockResourceQuotas).getNamespaceBundleResourceQuota("myprop/clust/ns1", "0x80000000_0xffffffff");
    cmdResourceQuotas.run(split("set --namespace myprop/clust/ns1 --bundle 0x80000000_0xffffffff -mi 10 -mo 20 -bi 10000 -bo 20000 -mem 100"));
    verify(mockResourceQuotas).setNamespaceBundleResourceQuota("myprop/clust/ns1", "0x80000000_0xffffffff", quota);
    cmdResourceQuotas.run(split("reset-namespace-bundle-quota --namespace myprop/clust/ns1 --bundle 0x80000000_0xffffffff"));
    verify(mockResourceQuotas).resetNamespaceBundleResourceQuota("myprop/clust/ns1", "0x80000000_0xffffffff");
}

Example 20 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method doLoadRanking.

/**
 * Rank brokers by available capacity, or load percentage, based on placement strategy:
 *
 * - Available capacity for weighted random selection (weightedRandomSelection): ranks ResourceUnits units based on
 * estimation of their capacity which is basically how many bundles each ResourceUnit is able can handle with its
 * available resources (CPU, memory, network, etc);
 *
 * - Load percentage for least loaded server (leastLoadedServer): ranks ResourceUnits units based on estimation of
 * their load percentage which is basically how many percent of resource is allocated which is
 * max(resource_actually_used, resource_quota)
 *
 * If we fail to collect the Load Reports OR fail to process them for the first time, it means the leader does not
 * have enough information to make a decision so we set it to ready when we collect and process the load reports
 * successfully the first time.
 */
private synchronized void doLoadRanking() {
    ResourceUnitRanking.setCpuUsageByMsgRate(this.realtimeCpuLoadFactor);
    String hostname = pulsar.getAdvertisedAddress();
    String strategy = this.getLoadBalancerPlacementStrategy();
    log.info("doLoadRanking - load balancing strategy: {}", strategy);
    if (!currentLoadReports.isEmpty()) {
        Map<Long, Set<ResourceUnit>> newSortedRankings = Maps.newTreeMap();
        Map<ResourceUnit, ResourceUnitRanking> newResourceUnitRankings = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            // calculate rankings
            Set<String> loadedBundles = loadReport.getBundles();
            Set<String> preAllocatedBundles = null;
            if (resourceUnitRankings.containsKey(resourceUnit)) {
                preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles();
                preAllocatedBundles.removeAll(loadedBundles);
            } else {
                preAllocatedBundles = new HashSet<>();
            }
            ResourceQuota allocatedQuota = getTotalAllocatedQuota(loadedBundles);
            ResourceQuota preAllocatedQuota = getTotalAllocatedQuota(preAllocatedBundles);
            ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), loadedBundles, allocatedQuota, preAllocatedBundles, preAllocatedQuota);
            newResourceUnitRankings.put(resourceUnit, ranking);
            // generated sorted ranking
            double loadPercentage = ranking.getEstimatedLoadPercentage();
            long maxCapacity = ranking.estimateMaxCapacity(pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota());
            long finalRank = 0;
            if (strategy.equals(LOADBALANCER_STRATEGY_LLS)) {
                finalRank = (long) loadPercentage;
            } else if (strategy.equals(LOADBALANCER_STRATEGY_LEAST_MSG)) {
                finalRank = (long) ranking.getEstimatedMessageRate();
            } else {
                double idleRatio = (100 - loadPercentage) / 100;
                finalRank = (long) (maxCapacity * idleRatio * idleRatio);
            }
            if (!newSortedRankings.containsKey(finalRank)) {
                newSortedRankings.put(finalRank, new HashSet<ResourceUnit>());
            }
            newSortedRankings.get(finalRank).add(entry.getKey());
            if (log.isDebugEnabled()) {
                log.debug("Added Resource Unit [{}] with Rank [{}]", entry.getKey().getResourceId(), finalRank);
            }
            // update metrics
            if (resourceUnit.getResourceId().contains(hostname)) {
                updateLoadBalancingMetrics(hostname, finalRank, ranking);
            }
        }
        updateBrokerToNamespaceToBundle();
        this.sortedRankings.set(newSortedRankings);
        this.resourceUnitRankings = newResourceUnitRankings;
    } else {
        log.info("Leader broker[{}] No ResourceUnits to rank this run, Using Old Ranking", pulsar.getWebServiceAddress());
    }
}