Examples with SystemResourceUsage com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage used on opensource projects

Example 6 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImpl method updateRealtimeResourceQuota.

private synchronized void updateRealtimeResourceQuota() {
    long memObjectGroupSize = 500;
    if (!currentLoadReports.isEmpty()) {
        long totalBundles = 0;
        long totalMemGroups = 0;
        double totalMsgRateIn = 0.0;
        double totalMsgRateOut = 0.0;
        double totalMsgRate = 0.0;
        double totalCpuUsage = 0.0;
        double totalMemoryUsage = 0.0;
        double totalBandwidthIn = 0.0;
        double totalBandwidthOut = 0.0;
        long loadReportTimestamp = -1;
        // update resource factors
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            LoadReport loadReport = entry.getValue();
            if (loadReport.getTimestamp() > loadReportTimestamp) {
                loadReportTimestamp = loadReport.getTimestamp();
            }
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                totalBundles++;
                NamespaceBundleStats stats = statsEntry.getValue();
                totalMemGroups += (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                totalBandwidthIn += stats.msgThroughputIn;
                totalBandwidthOut += stats.msgThroughputOut;
            }
            SystemResourceUsage resUsage = loadReport.getSystemResourceUsage();
            totalMsgRateIn += loadReport.getMsgRateIn();
            totalMsgRateOut += loadReport.getMsgRateOut();
            totalCpuUsage = totalCpuUsage + resUsage.getCpu().usage;
            totalMemoryUsage = totalMemoryUsage + resUsage.getMemory().usage;
        }
        totalMsgRate = totalMsgRateIn + totalMsgRateOut;
        long timePast = loadReportTimestamp - this.lastResourceQuotaUpdateTimestamp;
        this.lastResourceQuotaUpdateTimestamp = loadReportTimestamp;
        if (totalMsgRate > 1000 && totalMemGroups > 30) {
            this.realtimeCpuLoadFactor = timeSmoothValue(this.realtimeCpuLoadFactor, totalCpuUsage / totalMsgRate, RESOURCE_QUOTA_MIN_CPU_FACTOR, RESOURCE_QUOTA_MAX_CPU_FACTOR, timePast);
            this.realtimeMemoryLoadFactor = timeSmoothValue(this.realtimeMemoryLoadFactor, totalMemoryUsage / totalMemGroups, RESOURCE_QUOTA_MIN_MEM_FACTOR, RESOURCE_QUOTA_MAX_MEM_FACTOR, timePast);
        }
        // calculate average bundle
        if (totalBundles > 30 && this.realtimeAvgResourceQuota.getDynamic()) {
            ResourceQuota oldQuota = this.realtimeAvgResourceQuota;
            ResourceQuota newQuota = timeSmoothQuota(oldQuota, totalMsgRateIn / totalBundles, totalMsgRateOut / totalBundles, totalBandwidthIn / totalBundles, totalBandwidthOut / totalBundles, totalMemoryUsage / totalBundles, timePast);
            this.realtimeAvgResourceQuota = newQuota;
        }
        // update realtime quota for each bundle
        Map<String, ResourceQuota> newQuotas = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                String bundle = statsEntry.getKey();
                NamespaceBundleStats stats = statsEntry.getValue();
                long memGroupCount = (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                double newMemoryQuota = memGroupCount * this.realtimeMemoryLoadFactor;
                ResourceQuota oldQuota = getResourceQuota(bundle);
                ResourceQuota newQuota = timeSmoothQuota(oldQuota, stats.msgRateIn, stats.msgRateOut, stats.msgThroughputIn, stats.msgThroughputOut, newMemoryQuota, timePast);
                newQuotas.put(bundle, newQuota);
            }
        }
        this.realtimeResourceQuotas.set(newQuotas);
    }
}

Example 7 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImpl method generateLoadReport.

@Override
public LoadReport generateLoadReport() throws Exception {
    long timeSinceLastGenMillis = System.currentTimeMillis() - lastLoadReport.getTimestamp();
    if (timeSinceLastGenMillis <= LOAD_REPORT_UPDATE_MIMIMUM_INTERVAL) {
        return lastLoadReport;
    }
    try {
        LoadReport loadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
        loadReport.setName(String.format("%s:%s", pulsar.getAdvertisedAddress(), pulsar.getConfiguration().getWebServicePort()));
        SystemResourceUsage systemResourceUsage = this.getSystemResourceUsage();
        loadReport.setOverLoaded(isAboveLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerOverloadedThresholdPercentage()));
        loadReport.setUnderLoaded(isBelowLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerUnderloadedThresholdPercentage()));
        loadReport.setSystemResourceUsage(systemResourceUsage);
        loadReport.setBundleStats(pulsar.getBrokerService().getBundleStats());
        loadReport.setTimestamp(System.currentTimeMillis());
        return loadReport;
    } catch (Exception e) {
        log.error("[{}] Failed to generate LoadReport for broker, reason [{}]", e.getMessage(), e);
        throw e;
    }
}

Example 8 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImpl method getSystemResourceUsage.

private SystemResourceUsage getSystemResourceUsage() throws IOException {
    SystemResourceUsage systemResourceUsage = brokerHostUsage.getBrokerHostUsage();
    // Override System memory usage and limit with JVM heap usage and limit
    long maxHeapMemoryInBytes = Runtime.getRuntime().maxMemory();
    long memoryUsageInMBytes = getAverageJvmHeapUsageMBytes();
    systemResourceUsage.memory.usage = (double) memoryUsageInMBytes;
    systemResourceUsage.memory.limit = (double) (maxHeapMemoryInBytes) / MBytes;
    // Collect JVM direct memory
    systemResourceUsage.directMemory.usage = (double) (sun.misc.SharedSecrets.getJavaNioAccess().getDirectBufferPool().getMemoryUsed() / MBytes);
    systemResourceUsage.directMemory.limit = (double) (sun.misc.VM.maxDirectMemory() / MBytes);
    return systemResourceUsage;
}

Example 9 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class LoadBalancerTest method testNamespaceBundleAutoSplit.

/**
     * Test the namespace bundle auto-split
     */
@Test
public void testNamespaceBundleAutoSplit() throws Exception {
    int maxBundles = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxTopics = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    int maxSessions = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxMsgRate = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBandwidth = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1048576;
    pulsarServices[0].getConfiguration().setLoadBalancerAutoBundleSplitEnabled(true);
    // create namespaces
    for (int i = 1; i <= 10; i++) {
        int numBundles = (i == 10) ? maxBundles : 2;
        createNamespace(pulsarServices[0], String.format("pulsar/use/primary-ns-%02d", i), numBundles);
    }
    // fake Namespaces Admin
    NamespacesImpl namespaceAdmin = mock(NamespacesImpl.class);
    setObjectField(PulsarAdmin.class, pulsarServices[0].getAdminClient(), "namespaces", namespaceAdmin);
    // create load report
    // namespace 01~09 need to be split
    // namespace 08~10 don't need or cannot be split
    LoadReport lr = new LoadReport();
    lr.setName(lookupAddresses[0]);
    lr.setSystemResourceUsage(new SystemResourceUsage());
    Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
    bundleStats.put("pulsar/use/primary-ns-01/0x00000000_0x80000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-02/0x00000000_0x80000000", newBundleStats(2, maxSessions + 1, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-03/0x00000000_0x80000000", newBundleStats(2, 0, maxSessions + 1, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-04/0x00000000_0x80000000", newBundleStats(2, 0, 0, maxMsgRate + 1, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-05/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, maxMsgRate + 1, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-06/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, maxBandwidth + 1, 0));
    bundleStats.put("pulsar/use/primary-ns-07/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-08/0x00000000_0x80000000", newBundleStats(maxTopics - 1, maxSessions - 1, 1, maxMsgRate - 1, 1, maxBandwidth - 1, 1));
    bundleStats.put("pulsar/use/primary-ns-09/0x00000000_0x80000000", newBundleStats(1, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-10/0x00000000_0x02000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    lr.setBundleStats(bundleStats);
    setObjectField(SimpleLoadManagerImpl.class, pulsarServices[0].getLoadManager(), "lastLoadReport", lr);
    String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[0]);
    String loadReportJson = objectMapper.writeValueAsString(lr);
    bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    // sleep to wait load ranking be triggered and trigger bundle split
    Thread.sleep(5000);
    pulsarServices[0].getLoadManager().doNamespaceBundleSplit();
    // verify bundles are split
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-01", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-02", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-03", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-04", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-05", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-06", "0x00000000_0x80000000");
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-07", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-08", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-09", "0x00000000_0x80000000");
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-10", "0x00000000_0x02000000");
}

Example 10 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class LoadBalancerTest method testUpdateLoadReportAndCheckUpdatedRanking.

/*
     * tests rankings get updated when we write write the new load reports to the zookeeper on loadbalance root node
     * tests writing pre-configured load report on the zookeeper translates the pre-calculated rankings
     */
@Test
public void testUpdateLoadReportAndCheckUpdatedRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(256, 1024000));
        sru.setBandwidthOut(new ResourceUsage(250, 1024000));
        sru.setMemory(new ResourceUsage(1024, 8192));
        sru.setCpu(new ResourceUsage(5, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = objectMapper.writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait the load ranking be triggered
    Thread.sleep(5000);
    // do lookup for bunch of bundles
    int totalNamespaces = 200;
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        DestinationName fqdn = DestinationName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(fqdn));
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    // assert that distribution variation is not more than 10%
    int averageNamespaces = totalNamespaces / BROKER_COUNT;
    int tenPercentOfAverageNamespaces = averageNamespaces / 10;
    int lowerBound = averageNamespaces - tenPercentOfAverageNamespaces;
    int upperBound = averageNamespaces + tenPercentOfAverageNamespaces;
    // assert each broker received ownership of fair amount of namespaces 90%+
    for (Map.Entry<String, Integer> broker : namespaceOwner.entrySet()) {
        log.info("Count of bundles assigned: {}, {}", broker.getKey(), broker.getValue());
        assertTrue(broker.getValue() >= lowerBound && broker.getValue() <= upperBound);
    }
}