Examples with ResourceQuota org.apache.pulsar.common.policies.data.ResourceQuota used on opensource projects

Example 6 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method generateLoadReportForcefully.

private LoadReport generateLoadReportForcefully() throws Exception {
    synchronized (bundleGainsCache) {
        try {
            LoadReport loadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
            loadReport.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
            loadReport.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
            loadReport.setName(String.format("%s:%s", pulsar.getAdvertisedAddress(), pulsar.getConfiguration().getWebServicePort()));
            loadReport.setBrokerVersionString(pulsar.getBrokerVersion());
            SystemResourceUsage systemResourceUsage = this.getSystemResourceUsage();
            loadReport.setOverLoaded(isAboveLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerOverloadedThresholdPercentage()));
            loadReport.setUnderLoaded(isBelowLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerUnderloadedThresholdPercentage()));
            loadReport.setSystemResourceUsage(systemResourceUsage);
            loadReport.setBundleStats(pulsar.getBrokerService().getBundleStats());
            loadReport.setTimestamp(System.currentTimeMillis());
            final Set<String> oldBundles = lastLoadReport.getBundles();
            final Set<String> newBundles = loadReport.getBundles();
            bundleGainsCache.clear();
            bundleLossesCache.clear();
            for (String oldBundle : oldBundles) {
                if (!newBundles.contains(oldBundle)) {
                    bundleLossesCache.add(oldBundle);
                }
            }
            for (String newBundle : newBundles) {
                if (!oldBundles.contains(newBundle)) {
                    bundleGainsCache.add(newBundle);
                }
            }
            loadReport.setBundleGains(bundleGainsCache);
            loadReport.setBundleLosses(bundleLossesCache);
            final ResourceQuota allocatedQuota = getTotalAllocatedQuota(newBundles);
            loadReport.setAllocatedCPU((allocatedQuota.getMsgRateIn() + allocatedQuota.getMsgRateOut()) * realtimeCpuLoadFactor);
            loadReport.setAllocatedMemory(allocatedQuota.getMemory());
            loadReport.setAllocatedBandwidthIn(allocatedQuota.getBandwidthIn());
            loadReport.setAllocatedBandwidthOut(allocatedQuota.getBandwidthOut());
            loadReport.setAllocatedMsgRateIn(allocatedQuota.getMsgRateIn());
            loadReport.setAllocatedMsgRateOut(allocatedQuota.getMsgRateOut());
            final ResourceUnit resourceUnit = new SimpleResourceUnit(String.format("http://%s", loadReport.getName()), fromLoadReport(loadReport));
            Set<String> preAllocatedBundles;
            if (resourceUnitRankings.containsKey(resourceUnit)) {
                preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles();
                preAllocatedBundles.removeAll(newBundles);
            } else {
                preAllocatedBundles = new HashSet<>();
            }
            final ResourceQuota preAllocatedQuota = getTotalAllocatedQuota(preAllocatedBundles);
            loadReport.setPreAllocatedCPU((preAllocatedQuota.getMsgRateIn() + preAllocatedQuota.getMsgRateOut()) * realtimeCpuLoadFactor);
            loadReport.setPreAllocatedMemory(preAllocatedQuota.getMemory());
            loadReport.setPreAllocatedBandwidthIn(preAllocatedQuota.getBandwidthIn());
            loadReport.setPreAllocatedBandwidthOut(preAllocatedQuota.getBandwidthOut());
            loadReport.setPreAllocatedMsgRateIn(preAllocatedQuota.getMsgRateIn());
            loadReport.setPreAllocatedMsgRateOut(preAllocatedQuota.getMsgRateOut());
            return loadReport;
        } catch (Exception e) {
            log.error("[{}] Failed to generate LoadReport for broker, reason [{}]", e.getMessage(), e);
            throw e;
        }
    }
}

Example 7 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class ResourceQuotaCache method getQuota.

/**
 * Get resource quota for a specified service unit.
 *
 * @param suName
 *            identifier of the <code>ServiceUnit</code>
 *
 * @return the <code>ResourceQuota</code>.
 */
public ResourceQuota getQuota(String suName) {
    String zpath = ResourceQuotaCache.path(suName);
    ResourceQuota quota = this.readQuotaFromZnode(zpath);
    if (!quota.isValid()) {
        String defaultZpath = ResourceQuotaCache.path(null);
        quota = this.readQuotaFromZnode(defaultZpath);
        if (!quota.isValid()) {
            quota = ResourceQuotaCache.getInitialQuotaValue();
        }
    }
    return quota;
}

Example 8 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class LoadSimulationController method handleCopy.

// Handle the command line arguments associated with the copy command.
private void handleCopy(final ShellArguments arguments) throws Exception {
    final List<String> commandArguments = arguments.commandArguments;
    // Copy accepts 3 application arguments: Tenant name, source ZooKeeper and target ZooKeeper connect strings.
    if (checkAppArgs(commandArguments.size() - 1, 3)) {
        final String tenantName = commandArguments.get(1);
        final String sourceZKConnectString = commandArguments.get(2);
        final String targetZKConnectString = commandArguments.get(3);
        final ZooKeeper sourceZKClient = new ZooKeeper(sourceZKConnectString, 5000, null);
        final ZooKeeper targetZKClient = new ZooKeeper(targetZKConnectString, 5000, null);
        // Make a map for each thread to speed up the ZooKeeper writing process.
        final Map<String, ResourceQuota>[] threadLocalMaps = new Map[clients.length];
        for (int i = 0; i < clients.length; ++i) {
            threadLocalMaps[i] = new HashMap<>();
        }
        getResourceQuotas(QUOTA_ROOT, sourceZKClient, threadLocalMaps);
        final List<Future> futures = new ArrayList<>(clients.length);
        int i = 0;
        log.info("Copying...");
        for (final Map<String, ResourceQuota> bundleToQuota : threadLocalMaps) {
            final int j = i;
            futures.add(threadPool.submit(() -> {
                for (final Map.Entry<String, ResourceQuota> entry : bundleToQuota.entrySet()) {
                    final String bundle = entry.getKey();
                    final ResourceQuota quota = entry.getValue();
                    // Simulation will send messages in and out at about the same rate, so just make the rate the
                    // average of in and out.
                    final int tenantStart = QUOTA_ROOT.length() + 1;
                    final int clusterStart = bundle.indexOf('/', tenantStart) + 1;
                    final String sourceTenant = bundle.substring(tenantStart, clusterStart - 1);
                    final int namespaceStart = bundle.indexOf('/', clusterStart) + 1;
                    final String sourceCluster = bundle.substring(clusterStart, namespaceStart - 1);
                    final String namespace = bundle.substring(namespaceStart, bundle.lastIndexOf('/'));
                    final String keyRangeString = bundle.substring(bundle.lastIndexOf('/') + 1);
                    // To prevent duplicate node issues for same namespace names in different clusters/tenants.
                    final String manglePrefix = String.format("%s-%s-%s", sourceCluster, sourceTenant, keyRangeString);
                    final String mangledNamespace = String.format("%s-%s", manglePrefix, namespace);
                    final BundleData bundleData = initializeBundleData(quota, arguments);
                    final String oldAPITargetPath = String.format("/loadbalance/resource-quota/namespace/%s/%s/%s/0x00000000_0xffffffff", tenantName, cluster, mangledNamespace);
                    final String newAPITargetPath = String.format("/loadbalance/bundle-data/%s/%s/%s/0x00000000_0xffffffff", tenantName, cluster, mangledNamespace);
                    try {
                        ZkUtils.createFullPathOptimistic(targetZKClient, oldAPITargetPath, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(quota), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
                    } catch (KeeperException.NodeExistsException e) {
                    // Ignore already created nodes.
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                    // Put the bundle data in the new ZooKeeper.
                    try {
                        ZkUtils.createFullPathOptimistic(targetZKClient, newAPITargetPath, bundleData.getJsonBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
                    } catch (KeeperException.NodeExistsException e) {
                    // Ignore already created nodes.
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                    try {
                        trade(arguments, makeTopic(tenantName, mangledNamespace, "t"), j);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            }));
            ++i;
        }
        for (final Future future : futures) {
            future.get();
        }
        sourceZKClient.close();
        targetZKClient.close();
    }
}

Example 9 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class LoadBalancerTest method testDynamicNamespaceBundleQuota.

/*
     * Test broker dynamically calculating resource quota for each connected namespace bundle.
     */
@Test
public void testDynamicNamespaceBundleQuota() throws Exception {
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < BROKER_COUNT; i++) {
        ResourceQuota defaultQuota = new ResourceQuota();
        defaultQuota.setMsgRateIn(20);
        defaultQuota.setMsgRateOut(60);
        defaultQuota.setBandwidthIn(20000);
        defaultQuota.setBandwidthOut(60000);
        defaultQuota.setMemory(75);
        pulsarServices[i].getLocalZkCacheService().getResourceQuotaCache().setDefaultQuota(defaultQuota);
    }
    // publish the initial load reports and wait for quotas be updated
    writeLoadReportsForDynamicQuota(startTime);
    Thread.sleep(5000);
    // publish test report for 15 minutes later and wait for quotas be updated
    writeLoadReportsForDynamicQuota(startTime + SimpleLoadManagerImpl.RESOURCE_QUOTA_GO_UP_TIMEWINDOW);
    Thread.sleep(5000);
    // print & verify resource quotas
    for (int i = 0; i < BROKER_COUNT; i++) {
        Map<String, ResourceQuota> quotas = getRealtimeResourceQuota(pulsarServices[i]).get();
        printResourceQuotas(quotas);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-0-0/0x00000000_0xffffffff"), 19.0, 58.0, 19791.0, 58958.0, 74.0);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-2-2/0x00000000_0xffffffff"), 20.0, 60.0, 20000.0, 60000.0, 100.0);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-4-4/0x00000000_0xffffffff"), 40.0, 120.0, 40000.0, 120000.0, 150.0);
    }
    // publish test report for 24 hours later and wait for quotas be updated
    writeLoadReportsForDynamicQuota(startTime + SimpleLoadManagerImpl.RESOURCE_QUOTA_GO_DOWN_TIMEWINDOW);
    Thread.sleep(5000);
    // print & verify resource quotas
    for (int i = 0; i < BROKER_COUNT; i++) {
        Map<String, ResourceQuota> quotas = getRealtimeResourceQuota(pulsarServices[i]).get();
        printResourceQuotas(quotas);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-0-0/0x00000000_0xffffffff"), 5.0, 6.0, 10203.0, 11019.0, 50.0);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-2-2/0x00000000_0xffffffff"), 20.0, 60.0, 20000.0, 60000.0, 100.0);
        verifyBundleResourceQuota(quotas.get("pulsar/use/primary-ns-4-4/0x00000000_0xffffffff"), 40.0, 120.0, 40000.0, 120000.0, 150.0);
    }
}

Example 10 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class LoadBalancerTest method getRealtimeResourceQuota.

private AtomicReference<Map<String, ResourceQuota>> getRealtimeResourceQuota(PulsarService pulsar) throws NoSuchFieldException, IllegalAccessException {
    Field quotasField = ((SimpleLoadManagerImpl) pulsar.getLoadManager().get()).getClass().getDeclaredField("realtimeResourceQuotas");
    quotasField.setAccessible(true);
    AtomicReference<Map<String, ResourceQuota>> realtimeResourceQuotas = (AtomicReference<Map<String, ResourceQuota>>) quotasField.get(pulsar.getLoadManager().get());
    return realtimeResourceQuotas;
}