Examples with ResourceQuota org.apache.pulsar.common.policies.data.ResourceQuota used on opensource projects

Example 21 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method getResourceQuota.

private ResourceQuota getResourceQuota(String bundle) {
    Map<String, ResourceQuota> quotas = this.realtimeResourceQuotas.get();
    if (!quotas.containsKey(bundle)) {
        ResourceQuota quota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getQuota(bundle);
        quotas.put(bundle, quota);
        return quota;
    } else {
        return quotas.get(bundle);
    }
}

Example 22 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method writeResourceQuotasToZooKeeper.

@Override
public void writeResourceQuotasToZooKeeper() throws Exception {
    log.info("Writing namespace bundle resource quotas to ZooKeeper as leader broker");
    // write the load factors
    setDynamicConfigurationToZK(LOADBALANCER_DYNAMIC_SETTING_LOAD_FACTOR_CPU_ZPATH, new HashMap<String, String>() {

        {
            put(SETTING_NAME_LOAD_FACTOR_CPU, Double.toString(realtimeCpuLoadFactor));
        }
    });
    setDynamicConfigurationToZK(LOADBALANCER_DYNAMIC_SETTING_LOAD_FACTOR_MEM_ZPATH, new HashMap<String, String>() {

        {
            put(SETTING_NAME_LOAD_FACTOR_MEM, Double.toString(realtimeMemoryLoadFactor));
        }
    });
    // write default quota
    ResourceQuota defaultQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota();
    this.compareAndWriteQuota(null, defaultQuota, this.realtimeAvgResourceQuota);
    // write each bundle's quota
    Map<String, ResourceQuota> quotas = this.realtimeResourceQuotas.get();
    for (Map.Entry<String, ResourceQuota> entry : quotas.entrySet()) {
        String bundle = entry.getKey();
        ResourceQuota oldQuota = pulsar.getLocalZkCacheService().getResourceQuotaCache().getQuota(bundle);
        this.compareAndWriteQuota(bundle, oldQuota, entry.getValue());
    }
}

Example 23 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class LoadBalancerTest method testTopicAssignmentWithExistingBundles.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - already has some bundles
     * assigned Check the distribution of new topics is roughly consistent (with <10% variation) with the ranking
     */
@Test
public void testTopicAssignmentWithExistingBundles() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        ResourceQuota defaultQuota = new ResourceQuota();
        defaultQuota.setMsgRateIn(20);
        defaultQuota.setMsgRateOut(60);
        defaultQuota.setBandwidthIn(20000);
        defaultQuota.setBandwidthOut(60000);
        defaultQuota.setMemory(87);
        pulsarServices[i].getLocalZkCacheService().getResourceQuotaCache().setDefaultQuota(defaultQuota);
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(0, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
        for (int j = 0; j < (i + 1) * 5; j++) {
            String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
            NamespaceBundleStats stats = new NamespaceBundleStats();
            bundleStats.put(bundleName, stats);
        }
        lr.setBundleStats(bundleStats);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // print ranking
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
    }
    // check owner of new destiations and verify that the distribution is roughly
    // consistent (variation < 10%) with the broker capacity:
    int totalNamespaces = 250;
    int[] expectedAssignments = new int[] { 17, 34, 51, 68, 85 };
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        TopicName topicName = TopicName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().get().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(topicName)).get();
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    double expectedMaxVariation = 10.0;
    for (int i = 0; i < BROKER_COUNT; i++) {
        long actualValue = 0;
        String resourceId = "http://" + lookupAddresses[i];
        if (namespaceOwner.containsKey(resourceId)) {
            actualValue = namespaceOwner.get(resourceId);
        }
        long expectedValue = expectedAssignments[i];
        double variation = Math.abs(actualValue - expectedValue) * 100.0 / expectedValue;
        log.info("Topic assignment - {}, actual: {}, expected baseline: {}, variation: {}/%", lookupAddresses[i], actualValue, expectedValue, String.format("%.2f", variation));
        assertTrue(variation < expectedMaxVariation);
    }
}

Example 24 with ResourceQuota

use of org.apache.pulsar.common.policies.data.ResourceQuota in project incubator-pulsar by apache.

the class ResourceQuotaCacheTest method testGetSetBundleQuota.

@Test
public void testGetSetBundleQuota() throws Exception {
    ResourceQuotaCache cache = new ResourceQuotaCache(zkCache);
    NamespaceBundle testBundle = bundleFactory.getFullBundle(NamespaceName.get("pulsar/test/ns-2"));
    ResourceQuota quota1 = ResourceQuotaCache.getInitialQuotaValue();
    ResourceQuota quota2 = new ResourceQuota();
    quota2.setMsgRateIn(10);
    quota2.setMsgRateOut(20);
    quota2.setBandwidthIn(10000);
    quota2.setBandwidthOut(20000);
    quota2.setMemory(100);
    quota2.setDynamic(false);
    assertEquals(cache.getQuota(testBundle), quota1);
    cache.setQuota(testBundle, quota2);
    assertEquals(cache.getQuota(testBundle), quota2);
    cache.unsetQuota(testBundle);
    assertEquals(cache.getQuota(testBundle), quota1);
}