Examples with LoadReport com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport used on opensource projects

Example 11 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class DiscoveryServiceTest method addBrokerToZk.

private void addBrokerToZk(int number) throws Exception {
    for (int i = 0; i < number; i++) {
        LoadReport report = new LoadReport(null, null, "pulsar://broker-:15000" + i, null);
        String reportData = ObjectMapperFactory.getThreadLocal().writeValueAsString(report);
        ZkUtils.createFullPathOptimistic(mockZookKeeper, LOADBALANCE_BROKERS_ROOT + "/" + "broker-" + i, reportData.getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    }
    // wait to get cache updated
    Thread.sleep(100);
}

Example 12 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class LoadBalancerTest method testNamespaceBundleAutoSplit.

/**
     * Test the namespace bundle auto-split
     */
@Test
public void testNamespaceBundleAutoSplit() throws Exception {
    int maxBundles = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxTopics = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    int maxSessions = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxMsgRate = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBandwidth = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1048576;
    pulsarServices[0].getConfiguration().setLoadBalancerAutoBundleSplitEnabled(true);
    // create namespaces
    for (int i = 1; i <= 10; i++) {
        int numBundles = (i == 10) ? maxBundles : 2;
        createNamespace(pulsarServices[0], String.format("pulsar/use/primary-ns-%02d", i), numBundles);
    }
    // fake Namespaces Admin
    NamespacesImpl namespaceAdmin = mock(NamespacesImpl.class);
    setObjectField(PulsarAdmin.class, pulsarServices[0].getAdminClient(), "namespaces", namespaceAdmin);
    // create load report
    // namespace 01~09 need to be split
    // namespace 08~10 don't need or cannot be split
    LoadReport lr = new LoadReport();
    lr.setName(lookupAddresses[0]);
    lr.setSystemResourceUsage(new SystemResourceUsage());
    Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
    bundleStats.put("pulsar/use/primary-ns-01/0x00000000_0x80000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-02/0x00000000_0x80000000", newBundleStats(2, maxSessions + 1, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-03/0x00000000_0x80000000", newBundleStats(2, 0, maxSessions + 1, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-04/0x00000000_0x80000000", newBundleStats(2, 0, 0, maxMsgRate + 1, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-05/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, maxMsgRate + 1, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-06/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, maxBandwidth + 1, 0));
    bundleStats.put("pulsar/use/primary-ns-07/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-08/0x00000000_0x80000000", newBundleStats(maxTopics - 1, maxSessions - 1, 1, maxMsgRate - 1, 1, maxBandwidth - 1, 1));
    bundleStats.put("pulsar/use/primary-ns-09/0x00000000_0x80000000", newBundleStats(1, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-10/0x00000000_0x02000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    lr.setBundleStats(bundleStats);
    setObjectField(SimpleLoadManagerImpl.class, pulsarServices[0].getLoadManager(), "lastLoadReport", lr);
    String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[0]);
    String loadReportJson = objectMapper.writeValueAsString(lr);
    bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    // sleep to wait load ranking be triggered and trigger bundle split
    Thread.sleep(5000);
    pulsarServices[0].getLoadManager().doNamespaceBundleSplit();
    // verify bundles are split
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-01", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-02", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-03", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-04", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-05", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-06", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-07", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-08", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-09", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-10", "0x00000000_0x02000000");
}

Example 13 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class LoadBalancerTest method testUpdateLoadReportAndCheckUpdatedRanking.

/*
     * tests rankings get updated when we write write the new load reports to the zookeeper on loadbalance root node
     * tests writing pre-configured load report on the zookeeper translates the pre-calculated rankings
     */
@Test
public void testUpdateLoadReportAndCheckUpdatedRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(256, 1024000));
        sru.setBandwidthOut(new ResourceUsage(250, 1024000));
        sru.setMemory(new ResourceUsage(1024, 8192));
        sru.setCpu(new ResourceUsage(5, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = objectMapper.writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait the load ranking be triggered
    Thread.sleep(5000);
    // do lookup for bunch of bundles
    int totalNamespaces = 200;
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        DestinationName fqdn = DestinationName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(fqdn));
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    // assert that distribution variation is not more than 10%
    int averageNamespaces = totalNamespaces / BROKER_COUNT;
    int tenPercentOfAverageNamespaces = averageNamespaces / 10;
    int lowerBound = averageNamespaces - tenPercentOfAverageNamespaces;
    int upperBound = averageNamespaces + tenPercentOfAverageNamespaces;
    // assert each broker received ownership of fair amount of namespaces 90%+
    for (Map.Entry<String, Integer> broker : namespaceOwner.entrySet()) {
        log.info("Count of bundles assigned: {}, {}", broker.getKey(), broker.getValue());
        assertTrue(broker.getValue() >= lowerBound && broker.getValue() <= upperBound);
    }
}

Example 14 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class SimpleLoadManagerImpl method doLoadShedding.

@Override
public void doLoadShedding() {
    long overloadThreshold = this.getLoadBalancerBrokerOverloadedThresholdPercentage();
    long comfortLoadLevel = this.getLoadBalancerBrokerComfortLoadThresholdPercentage();
    log.info("Running load shedding task as leader broker, overload threshold {}, comfort loadlevel {}", overloadThreshold, comfortLoadLevel);
    // overloadedRU --> bundleName
    Map<ResourceUnit, String> namespaceBundlesToBeUnloaded = new HashMap<>();
    synchronized (currentLoadReports) {
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit overloadedRU = entry.getKey();
            LoadReport lr = entry.getValue();
            if (isAboveLoadLevel(lr.getSystemResourceUsage(), overloadThreshold)) {
                ResourceType bottleneckResourceType = lr.getBottleneckResourceType();
                Map<String, NamespaceBundleStats> bundleStats = lr.getSortedBundleStats(bottleneckResourceType);
                // 1. owns only one namespace
                if (bundleStats.size() == 1) {
                    // can't unload one namespace, just issue a warning message
                    String bundleName = lr.getBundleStats().keySet().iterator().next();
                    log.warn("HIGH USAGE WARNING : Sole namespace bundle {} is overloading broker {}. " + "No Load Shedding will be done on this broker", bundleName, overloadedRU.getResourceId());
                    continue;
                }
                for (Map.Entry<String, NamespaceBundleStats> bundleStat : bundleStats.entrySet()) {
                    String bundleName = bundleStat.getKey();
                    NamespaceBundleStats stats = bundleStat.getValue();
                    // We need at least one underloaded RU from list of candidates that can host this bundle
                    if (isBrokerAvailableForRebalancing(bundleStat.getKey(), comfortLoadLevel)) {
                        log.info("Namespace bundle {} will be unloaded from overloaded broker {}, bundle stats (topics: {}, producers {}, " + "consumers {}, bandwidthIn {}, bandwidthOut {})", bundleName, overloadedRU.getResourceId(), stats.topics, stats.producerCount, stats.consumerCount, stats.msgThroughputIn, stats.msgThroughputOut);
                        namespaceBundlesToBeUnloaded.put(overloadedRU, bundleName);
                    } else {
                        log.info("Unable to shed load from broker {}, no brokers with enough capacity available " + "for re-balancing {}", overloadedRU.getResourceId(), bundleName);
                    }
                    break;
                }
            }
        }
    }
    unloadNamespacesFromOverLoadedBrokers(namespaceBundlesToBeUnloaded);
}

Example 15 with LoadReport

use of com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport in project pulsar by yahoo.

the class NamespaceService method createLookupResult.

private CompletableFuture<LookupResult> createLookupResult(String candidateBroker) throws Exception {
    CompletableFuture<LookupResult> lookupFuture = new CompletableFuture<>();
    try {
        checkArgument(StringUtils.isNotBlank(candidateBroker), "Lookup broker can't be null " + candidateBroker);
        URI uri = new URI(candidateBroker);
        String path = String.format("%s/%s:%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, uri.getHost(), uri.getPort());
        pulsar.getLocalZkCache().getDataAsync(path, loadReportDeserializer).thenAccept(reportData -> {
            if (reportData.isPresent()) {
                LoadReport report = reportData.get();
                lookupFuture.complete(new LookupResult(report.getWebServiceUrl(), report.getWebServiceUrlTls(), report.getPulsarServiceUrl(), report.getPulsarServieUrlTls()));
            } else {
                lookupFuture.completeExceptionally(new KeeperException.NoNodeException(path));
            }
        }).exceptionally(ex -> {
            lookupFuture.completeExceptionally(ex);
            return null;
        });
    } catch (Exception e) {
        lookupFuture.completeExceptionally(e);
    }
    return lookupFuture;
}