Examples with SystemResourceUsage com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage used on opensource projects

Example 16 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class PulsarLoadReportImpl method parse.

public static LoadReport parse(String loadReportJson) {
    PulsarLoadReportImpl pulsarLoadReport = new PulsarLoadReportImpl();
    ObjectMapper mapper = ObjectMapperFactory.create();
    try {
        com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport report = mapper.readValue(loadReportJson, com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport.class);
        SystemResourceUsage sru = report.getSystemResourceUsage();
        String resourceUnitName = report.getName();
        pulsarLoadReport.resourceDescription = new PulsarResourceDescription();
        if (sru.bandwidthIn != null)
            pulsarLoadReport.resourceDescription.put("bandwidthIn", sru.bandwidthIn);
        if (sru.bandwidthOut != null)
            pulsarLoadReport.resourceDescription.put("bandwidthOut", sru.bandwidthOut);
        if (sru.memory != null)
            pulsarLoadReport.resourceDescription.put("memory", sru.memory);
        if (sru.cpu != null)
            pulsarLoadReport.resourceDescription.put("cpu", sru.cpu);
        pulsarLoadReport.resourceUnit = new SimpleResourceUnit(resourceUnitName, pulsarLoadReport.resourceDescription);
    } catch (Exception e) {
        log.warn("Failed Parsing Load Report from JSON string [{}]", e);
    }
    return pulsarLoadReport;
}

Example 17 with SystemResourceUsage

use of com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage in project pulsar by yahoo.

the class SimpleLoadManagerImpl method writeLoadReportOnZookeeper.

@Override
public void writeLoadReportOnZookeeper() throws Exception {
    // update average JVM heap usage to average value of the last 120 seconds
    long realtimeJvmHeapUsage = getRealtimeJvmHeapUsageMBytes();
    if (this.avgJvmHeapUsageMBytes <= 0) {
        this.avgJvmHeapUsageMBytes = realtimeJvmHeapUsage;
    } else {
        long weight = Math.max(1, TimeUnit.SECONDS.toMillis(120) / LOAD_REPORT_UPDATE_MIMIMUM_INTERVAL);
        this.avgJvmHeapUsageMBytes = ((weight - 1) * this.avgJvmHeapUsageMBytes + realtimeJvmHeapUsage) / weight;
    }
    // Update LoadReport in below situations:
    // 1) This is the first time to update LoadReport
    // 2) The last LoadReport is 5 minutes ago
    // 3) There is more than 10% change on number of bundles assigned comparing with broker's maximum capacity
    // 4) There is more than 10% change on resource usage comparing with broker's resource limit
    boolean needUpdate = false;
    if (lastLoadReport == null || this.forceLoadReportUpdate == true) {
        needUpdate = true;
        this.forceLoadReportUpdate = false;
    } else {
        long timestampNow = System.currentTimeMillis();
        long timeElapsedSinceLastReport = timestampNow - lastLoadReport.getTimestamp();
        int maxUpdateIntervalInMinutes = pulsar.getConfiguration().getLoadBalancerReportUpdateMaxIntervalMinutes();
        if (timeElapsedSinceLastReport > TimeUnit.MINUTES.toMillis(maxUpdateIntervalInMinutes)) {
            needUpdate = true;
        } else if (timeElapsedSinceLastReport > LOAD_REPORT_UPDATE_MIMIMUM_INTERVAL) {
            // check number of bundles assigned, comparing with last LoadReport
            long oldBundleCount = lastLoadReport.getNumBundles();
            long newBundleCount = pulsar.getBrokerService().getNumberOfNamespaceBundles();
            long bundleCountChange = Math.abs(oldBundleCount - newBundleCount);
            long maxCapacity = ResourceUnitRanking.calculateBrokerMaxCapacity(lastLoadReport.getSystemResourceUsage(), pulsar.getLocalZkCacheService().getResourceQuotaCache().getDefaultQuota());
            double bundlePercentageChange = (maxCapacity > 0) ? (bundleCountChange * 100 / maxCapacity) : 0;
            if (newBundleCount < oldBundleCount || bundlePercentageChange > pulsar.getConfiguration().getLoadBalancerReportUpdateThresholdPercentage()) {
                needUpdate = true;
            }
            // check resource usage comparing with last LoadReport
            if (!needUpdate && timestampNow - this.lastResourceUsageTimestamp > TimeUnit.MINUTES.toMillis(pulsar.getConfiguration().getLoadBalancerHostUsageCheckIntervalMinutes())) {
                SystemResourceUsage oldUsage = lastLoadReport.getSystemResourceUsage();
                SystemResourceUsage newUsage = this.getSystemResourceUsage();
                this.lastResourceUsageTimestamp = timestampNow;
                // calculate percentage of change
                double cpuChange = (newUsage.cpu.limit > 0) ? ((newUsage.cpu.usage - oldUsage.cpu.usage) * 100 / newUsage.cpu.limit) : 0;
                double memChange = (newUsage.memory.limit > 0) ? ((newUsage.memory.usage - oldUsage.memory.usage) * 100 / newUsage.memory.limit) : 0;
                double directMemChange = (newUsage.directMemory.limit > 0) ? ((newUsage.directMemory.usage - oldUsage.directMemory.usage) * 100 / newUsage.directMemory.limit) : 0;
                double bandwidthOutChange = (newUsage.bandwidthOut.limit > 0) ? ((newUsage.bandwidthOut.usage - oldUsage.bandwidthOut.usage) * 100 / newUsage.bandwidthOut.limit) : 0;
                double bandwidthInChange = (newUsage.bandwidthIn.limit > 0) ? ((newUsage.bandwidthIn.usage - oldUsage.bandwidthIn.usage) * 100 / newUsage.bandwidthIn.limit) : 0;
                long resourceChange = (long) Math.min(100.0, Math.max(Math.abs(cpuChange), Math.max(Math.abs(directMemChange), Math.max(Math.abs(memChange), Math.max(Math.abs(bandwidthOutChange), Math.abs(bandwidthInChange))))));
                if (resourceChange > pulsar.getConfiguration().getLoadBalancerReportUpdateThresholdPercentage()) {
                    needUpdate = true;
                    log.info("LoadReport update triggered by change on resource usage, detal ({}).", String.format("cpu: %.1f%%, mem: %.1f%%, directMemory: %.1f%%, bandwidthIn: %.1f%%, bandwidthOut: %.1f%%)", cpuChange, memChange, directMemChange, bandwidthInChange, bandwidthOutChange));
                }
            }
        }
    }
    if (needUpdate) {
        LoadReport lr = generateLoadReport();
        pulsar.getZkClient().setData(brokerZnodePath, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(lr), -1);
        this.lastLoadReport = lr;
        this.lastResourceUsageTimestamp = lr.getTimestamp();
        // split-bundle if requires
        doNamespaceBundleSplit();
    }
}