Examples with SystemResourceUsage org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage used on opensource projects

Example 6 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method updateRealtimeResourceQuota.

private synchronized void updateRealtimeResourceQuota() {
    long memObjectGroupSize = 500;
    if (!currentLoadReports.isEmpty()) {
        long totalBundles = 0;
        long totalMemGroups = 0;
        double totalMsgRateIn = 0.0;
        double totalMsgRateOut = 0.0;
        double totalMsgRate = 0.0;
        double totalCpuUsage = 0.0;
        double totalMemoryUsage = 0.0;
        double totalBandwidthIn = 0.0;
        double totalBandwidthOut = 0.0;
        long loadReportTimestamp = -1;
        // update resource factors
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            LoadReport loadReport = entry.getValue();
            if (loadReport.getTimestamp() > loadReportTimestamp) {
                loadReportTimestamp = loadReport.getTimestamp();
            }
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                totalBundles++;
                NamespaceBundleStats stats = statsEntry.getValue();
                totalMemGroups += (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                totalBandwidthIn += stats.msgThroughputIn;
                totalBandwidthOut += stats.msgThroughputOut;
            }
            SystemResourceUsage resUsage = loadReport.getSystemResourceUsage();
            totalMsgRateIn += loadReport.getMsgRateIn();
            totalMsgRateOut += loadReport.getMsgRateOut();
            totalCpuUsage = totalCpuUsage + resUsage.getCpu().usage;
            totalMemoryUsage = totalMemoryUsage + resUsage.getMemory().usage;
        }
        totalMsgRate = totalMsgRateIn + totalMsgRateOut;
        long timePast = loadReportTimestamp - this.lastResourceQuotaUpdateTimestamp;
        this.lastResourceQuotaUpdateTimestamp = loadReportTimestamp;
        if (totalMsgRate > 1000 && totalMemGroups > 30) {
            this.realtimeCpuLoadFactor = timeSmoothValue(this.realtimeCpuLoadFactor, totalCpuUsage / totalMsgRate, RESOURCE_QUOTA_MIN_CPU_FACTOR, RESOURCE_QUOTA_MAX_CPU_FACTOR, timePast);
            this.realtimeMemoryLoadFactor = timeSmoothValue(this.realtimeMemoryLoadFactor, totalMemoryUsage / totalMemGroups, RESOURCE_QUOTA_MIN_MEM_FACTOR, RESOURCE_QUOTA_MAX_MEM_FACTOR, timePast);
        }
        // calculate average bundle
        if (totalBundles > 30 && this.realtimeAvgResourceQuota.getDynamic()) {
            ResourceQuota oldQuota = this.realtimeAvgResourceQuota;
            ResourceQuota newQuota = timeSmoothQuota(oldQuota, totalMsgRateIn / totalBundles, totalMsgRateOut / totalBundles, totalBandwidthIn / totalBundles, totalBandwidthOut / totalBundles, totalMemoryUsage / totalBundles, timePast);
            this.realtimeAvgResourceQuota = newQuota;
        }
        // update realtime quota for each bundle
        Map<String, ResourceQuota> newQuotas = new HashMap<>();
        for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
            ResourceUnit resourceUnit = entry.getKey();
            LoadReport loadReport = entry.getValue();
            Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
            if (bundleStats == null) {
                continue;
            }
            for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
                String bundle = statsEntry.getKey();
                NamespaceBundleStats stats = statsEntry.getValue();
                long memGroupCount = (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
                double newMemoryQuota = memGroupCount * this.realtimeMemoryLoadFactor;
                ResourceQuota oldQuota = getResourceQuota(bundle);
                ResourceQuota newQuota = timeSmoothQuota(oldQuota, stats.msgRateIn, stats.msgRateOut, stats.msgThroughputIn, stats.msgThroughputOut, newMemoryQuota, timePast);
                newQuotas.put(bundle, newQuota);
            }
        }
        this.realtimeResourceQuotas.set(newQuotas);
    }
}

Example 7 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method generateLoadReportForcefully.

private LoadReport generateLoadReportForcefully() throws Exception {
    synchronized (bundleGainsCache) {
        try {
            LoadReport loadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
            loadReport.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
            loadReport.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
            loadReport.setName(String.format("%s:%s", pulsar.getAdvertisedAddress(), pulsar.getConfiguration().getWebServicePort()));
            loadReport.setBrokerVersionString(pulsar.getBrokerVersion());
            SystemResourceUsage systemResourceUsage = this.getSystemResourceUsage();
            loadReport.setOverLoaded(isAboveLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerOverloadedThresholdPercentage()));
            loadReport.setUnderLoaded(isBelowLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerUnderloadedThresholdPercentage()));
            loadReport.setSystemResourceUsage(systemResourceUsage);
            loadReport.setBundleStats(pulsar.getBrokerService().getBundleStats());
            loadReport.setTimestamp(System.currentTimeMillis());
            final Set<String> oldBundles = lastLoadReport.getBundles();
            final Set<String> newBundles = loadReport.getBundles();
            bundleGainsCache.clear();
            bundleLossesCache.clear();
            for (String oldBundle : oldBundles) {
                if (!newBundles.contains(oldBundle)) {
                    bundleLossesCache.add(oldBundle);
                }
            }
            for (String newBundle : newBundles) {
                if (!oldBundles.contains(newBundle)) {
                    bundleGainsCache.add(newBundle);
                }
            }
            loadReport.setBundleGains(bundleGainsCache);
            loadReport.setBundleLosses(bundleLossesCache);
            final ResourceQuota allocatedQuota = getTotalAllocatedQuota(newBundles);
            loadReport.setAllocatedCPU((allocatedQuota.getMsgRateIn() + allocatedQuota.getMsgRateOut()) * realtimeCpuLoadFactor);
            loadReport.setAllocatedMemory(allocatedQuota.getMemory());
            loadReport.setAllocatedBandwidthIn(allocatedQuota.getBandwidthIn());
            loadReport.setAllocatedBandwidthOut(allocatedQuota.getBandwidthOut());
            loadReport.setAllocatedMsgRateIn(allocatedQuota.getMsgRateIn());
            loadReport.setAllocatedMsgRateOut(allocatedQuota.getMsgRateOut());
            final ResourceUnit resourceUnit = new SimpleResourceUnit(String.format("http://%s", loadReport.getName()), fromLoadReport(loadReport));
            Set<String> preAllocatedBundles;
            if (resourceUnitRankings.containsKey(resourceUnit)) {
                preAllocatedBundles = resourceUnitRankings.get(resourceUnit).getPreAllocatedBundles();
                preAllocatedBundles.removeAll(newBundles);
            } else {
                preAllocatedBundles = new HashSet<>();
            }
            final ResourceQuota preAllocatedQuota = getTotalAllocatedQuota(preAllocatedBundles);
            loadReport.setPreAllocatedCPU((preAllocatedQuota.getMsgRateIn() + preAllocatedQuota.getMsgRateOut()) * realtimeCpuLoadFactor);
            loadReport.setPreAllocatedMemory(preAllocatedQuota.getMemory());
            loadReport.setPreAllocatedBandwidthIn(preAllocatedQuota.getBandwidthIn());
            loadReport.setPreAllocatedBandwidthOut(preAllocatedQuota.getBandwidthOut());
            loadReport.setPreAllocatedMsgRateIn(preAllocatedQuota.getMsgRateIn());
            loadReport.setPreAllocatedMsgRateOut(preAllocatedQuota.getMsgRateOut());
            return loadReport;
        } catch (Exception e) {
            log.error("[{}] Failed to generate LoadReport for broker, reason [{}]", e.getMessage(), e);
            throw e;
        }
    }
}

Example 8 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method writeLoadReportOnZookeeper.

@Override
public void writeLoadReportOnZookeeper() throws Exception {
    // update average JVM heap usage to average value of the last 120 seconds
    long realtimeJvmHeapUsage = getRealtimeJvmHeapUsageMBytes();
    if (this.avgJvmHeapUsageMBytes <= 0) {
        this.avgJvmHeapUsageMBytes = realtimeJvmHeapUsage;
    } else {
        long weight = Math.max(1, TimeUnit.SECONDS.toMillis(120) / LOAD_REPORT_UPDATE_MIMIMUM_INTERVAL);
        this.avgJvmHeapUsageMBytes = ((weight - 1) * this.avgJvmHeapUsageMBytes + realtimeJvmHeapUsage) / weight;
    }
    // Update LoadReport in below situations:
    // 1) This is the first time to update LoadReport
    // 2) The last LoadReport is 5 minutes ago
    // 3) There is more than 10% change on number of bundles assigned comparing with broker's maximum capacity
    // 4) There is more than 10% change on resource usage comparing with broker's resource limit
    boolean needUpdate = false;
    if (lastLoadReport == null || this.forceLoadReportUpdate == true) {
        needUpdate = true;
        this.forceLoadReportUpdate = false;
    } else {
        long timestampNow = System.currentTimeMillis();
        long timeElapsedSinceLastReport = timestampNow - lastLoadReport.getTimestamp();
        int maxUpdateIntervalInMinutes = pulsar.getConfiguration().getLoadBalancerReportUpdateMaxIntervalMinutes();
        if (timeElapsedSinceLastReport > TimeUnit.MINUTES.toMillis(maxUpdateIntervalInMinutes)) {
            needUpdate = true;
        } else if (timeElapsedSinceLastReport > LOAD_REPORT_UPDATE_MIMIMUM_INTERVAL) {
            // check number of bundles assigned, comparing with last LoadReport
            long oldBundleCount = lastLoadReport.getNumBundles();
            long newBundleCount = pulsar.getBrokerService().getNumberOfNamespaceBundles();
            long bundleCountChange = Math.abs(oldBundleCount - newBundleCount);
            long maxCapacity = ResourceUnitRanking.calculateBrokerMaxCapacity(lastLoadReport.getSystemResourceUsage(), pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota());
            double bundlePercentageChange = (maxCapacity > 0) ? (bundleCountChange * 100 / maxCapacity) : 0;
            if (newBundleCount != oldBundleCount) {
                needUpdate = true;
            }
            // check resource usage comparing with last LoadReport
            if (!needUpdate && timestampNow - this.lastResourceUsageTimestamp > TimeUnit.MINUTES.toMillis(pulsar.getConfiguration().getLoadBalancerHostUsageCheckIntervalMinutes())) {
                SystemResourceUsage oldUsage = lastLoadReport.getSystemResourceUsage();
                SystemResourceUsage newUsage = this.getSystemResourceUsage();
                this.lastResourceUsageTimestamp = timestampNow;
                // calculate percentage of change
                double cpuChange = (newUsage.cpu.limit > 0) ? ((newUsage.cpu.usage - oldUsage.cpu.usage) * 100 / newUsage.cpu.limit) : 0;
                double memChange = (newUsage.memory.limit > 0) ? ((newUsage.memory.usage - oldUsage.memory.usage) * 100 / newUsage.memory.limit) : 0;
                double directMemChange = (newUsage.directMemory.limit > 0) ? ((newUsage.directMemory.usage - oldUsage.directMemory.usage) * 100 / newUsage.directMemory.limit) : 0;
                double bandwidthOutChange = (newUsage.bandwidthOut.limit > 0) ? ((newUsage.bandwidthOut.usage - oldUsage.bandwidthOut.usage) * 100 / newUsage.bandwidthOut.limit) : 0;
                double bandwidthInChange = (newUsage.bandwidthIn.limit > 0) ? ((newUsage.bandwidthIn.usage - oldUsage.bandwidthIn.usage) * 100 / newUsage.bandwidthIn.limit) : 0;
                long resourceChange = (long) Math.min(100.0, Math.max(Math.abs(cpuChange), Math.max(Math.abs(directMemChange), Math.max(Math.abs(memChange), Math.max(Math.abs(bandwidthOutChange), Math.abs(bandwidthInChange))))));
                if (resourceChange > pulsar.getConfiguration().getLoadBalancerReportUpdateThresholdPercentage()) {
                    needUpdate = true;
                    log.info("LoadReport update triggered by change on resource usage, detal ({}).", String.format("cpu: %.1f%%, mem: %.1f%%, directMemory: %.1f%%, bandwidthIn: %.1f%%, bandwidthOut: %.1f%%)", cpuChange, memChange, directMemChange, bandwidthInChange, bandwidthOutChange));
                }
            }
        }
    }
    if (needUpdate) {
        LoadReport lr = generateLoadReportForcefully();
        pulsar.getZkClient().setData(brokerZnodePath, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(lr), -1);
        this.lastLoadReport = lr;
        this.lastResourceUsageTimestamp = lr.getTimestamp();
        // split-bundle if requires
        doNamespaceBundleSplit();
    }
}

Example 9 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class LookupDataTest method testLoadReportSerialization.

@Test
public void testLoadReportSerialization() throws Exception {
    final String simpleLmBrokerUrl = "simple";
    final String simpleLmReportName = "simpleLoadManager";
    final String modularLmBrokerUrl = "modular";
    final SystemResourceUsage simpleLmSystemResourceUsage = new SystemResourceUsage();
    final ResourceUsage resource = new ResourceUsage();
    final double usage = 55.0;
    resource.usage = usage;
    simpleLmSystemResourceUsage.bandwidthIn = resource;
    LoadReport simpleReport = getSimpleLoadManagerLoadReport(simpleLmBrokerUrl, simpleLmReportName, simpleLmSystemResourceUsage);
    LocalBrokerData modularReport = getModularLoadManagerLoadReport(modularLmBrokerUrl, resource);
    LoadManagerReport simpleLoadReport = ObjectMapperFactory.getThreadLocal().readValue(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(simpleReport), LoadManagerReport.class);
    LoadManagerReport modularLoadReport = ObjectMapperFactory.getThreadLocal().readValue(ObjectMapperFactory.getThreadLocal().writeValueAsBytes(modularReport), LoadManagerReport.class);
    assertEquals(simpleLoadReport.getWebServiceUrl(), simpleLmBrokerUrl);
    assertTrue(simpleLoadReport instanceof LoadReport);
    assertEquals(((LoadReport) simpleLoadReport).getName(), simpleLmReportName);
    assertEquals(((LoadReport) simpleLoadReport).getSystemResourceUsage().bandwidthIn.usage, usage);
    assertEquals(modularLoadReport.getWebServiceUrl(), modularLmBrokerUrl);
    assertTrue(modularLoadReport instanceof LocalBrokerData);
    assertEquals(((LocalBrokerData) modularLoadReport).getBandwidthIn().usage, usage);
}

Example 10 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class BrokerMonitor method printGlobalData.

// Prints out the global load data.
private void printGlobalData() {
    synchronized (loadData) {
        // 1 header row, 1 total row, and loadData.size() rows for brokers.
        Object[][] rows = new Object[loadData.size() + 2][];
        rows[0] = GLOBAL_HEADER;
        int totalBundles = 0;
        double totalThroughput = 0;
        double totalMessageRate = 0;
        double totalLongTermMessageRate = 0;
        double maxMaxUsage = 0;
        int i = 1;
        for (final Map.Entry<String, Object> entry : loadData.entrySet()) {
            final String broker = entry.getKey();
            final Object data = entry.getValue();
            rows[i] = new Object[GLOBAL_HEADER.length];
            rows[i][0] = broker;
            int numBundles;
            double messageRate;
            double longTermMessageRate;
            double messageThroughput;
            double maxUsage;
            if (data instanceof LoadReport) {
                final LoadReport loadReport = (LoadReport) data;
                numBundles = (int) loadReport.getNumBundles();
                messageRate = loadReport.getMsgRateIn() + loadReport.getMsgRateOut();
                longTermMessageRate = loadReport.getAllocatedMsgRateIn() + loadReport.getAllocatedMsgRateOut();
                messageThroughput = (loadReport.getAllocatedBandwidthIn() + loadReport.getAllocatedBandwidthOut()) / 1024;
                final SystemResourceUsage systemResourceUsage = loadReport.getSystemResourceUsage();
                maxUsage = Math.max(Math.max(Math.max(systemResourceUsage.getCpu().percentUsage(), systemResourceUsage.getMemory().percentUsage()), Math.max(systemResourceUsage.getDirectMemory().percentUsage(), systemResourceUsage.getBandwidthIn().percentUsage())), systemResourceUsage.getBandwidthOut().percentUsage());
            } else if (data instanceof LocalBrokerData) {
                final LocalBrokerData localData = (LocalBrokerData) data;
                numBundles = localData.getNumBundles();
                messageRate = localData.getMsgRateIn() + localData.getMsgRateOut();
                final String timeAveragePath = ModularLoadManagerImpl.TIME_AVERAGE_BROKER_ZPATH + "/" + broker;
                try {
                    final TimeAverageBrokerData timeAverageData = gson.fromJson(new String(zkClient.getData(timeAveragePath, false, null)), TimeAverageBrokerData.class);
                    longTermMessageRate = timeAverageData.getLongTermMsgRateIn() + timeAverageData.getLongTermMsgRateOut();
                } catch (Exception x) {
                    throw new RuntimeException(x);
                }
                messageThroughput = (localData.getMsgThroughputIn() + localData.getMsgThroughputOut()) / 1024;
                maxUsage = localData.getMaxResourceUsage();
            } else {
                throw new AssertionError("Unreachable code");
            }
            rows[i][1] = numBundles;
            rows[i][2] = messageRate;
            rows[i][3] = messageThroughput;
            rows[i][4] = longTermMessageRate;
            rows[i][5] = maxUsage;
            totalBundles += numBundles;
            totalMessageRate += messageRate;
            totalLongTermMessageRate += longTermMessageRate;
            totalThroughput += messageThroughput;
            maxMaxUsage = Math.max(maxUsage, maxMaxUsage);
            ++i;
        }
        final int finalRow = loadData.size() + 1;
        rows[finalRow] = new Object[GLOBAL_HEADER.length];
        rows[finalRow][0] = "TOTAL";
        rows[finalRow][1] = totalBundles;
        rows[finalRow][2] = totalMessageRate;
        rows[finalRow][3] = totalLongTermMessageRate;
        rows[finalRow][4] = totalThroughput;
        rows[finalRow][5] = maxMaxUsage;
        final String table = globalTableMaker.make(rows);
        log.info("Overall Broker Data:\n{}", table);
    }
}