Examples with TimeAverageBrokerData org.apache.pulsar.broker.TimeAverageBrokerData used on opensource projects

Example 1 with TimeAverageBrokerData

use of org.apache.pulsar.broker.TimeAverageBrokerData in project incubator-pulsar by apache.

the class LeastLongTermMessageRate method getScore.

// Form a score for a broker using its preallocated bundle data and time average data.
// This is done by summing all preallocated long-term message rates and adding them to the broker's overall
// long-term message rate, which is itself the sum of the long-term message rate of every allocated bundle.
// Any broker at (or above) the overload threshold will have a score of POSITIVE_INFINITY.
private static double getScore(final BrokerData brokerData, final ServiceConfiguration conf) {
    final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0;
    final double maxUsage = brokerData.getLocalData().getMaxResourceUsage();
    if (maxUsage > overloadThreshold) {
        log.warn("Broker {} is overloaded: max usage={}", brokerData.getLocalData().getWebServiceUrl(), maxUsage);
        return Double.POSITIVE_INFINITY;
    }
    double totalMessageRate = 0;
    for (BundleData bundleData : brokerData.getPreallocatedBundleData().values()) {
        final TimeAverageMessageData longTermData = bundleData.getLongTermData();
        totalMessageRate += longTermData.getMsgRateIn() + longTermData.getMsgRateOut();
    }
    // calculate estimated score
    final TimeAverageBrokerData timeAverageData = brokerData.getTimeAverageData();
    final double timeAverageLongTermMessageRate = timeAverageData.getLongTermMsgRateIn() + timeAverageData.getLongTermMsgRateOut();
    final double totalMessageRateEstimate = totalMessageRate + timeAverageLongTermMessageRate;
    if (log.isDebugEnabled()) {
        log.debug("Broker {} has long term message rate {}", brokerData.getLocalData().getWebServiceUrl(), totalMessageRateEstimate);
    }
    return totalMessageRateEstimate;
}

Example 2 with TimeAverageBrokerData

use of org.apache.pulsar.broker.TimeAverageBrokerData in project incubator-pulsar by apache.

the class BrokerMonitor method printGlobalData.

// Prints out the global load data.
private void printGlobalData() {
    synchronized (loadData) {
        // 1 header row, 1 total row, and loadData.size() rows for brokers.
        Object[][] rows = new Object[loadData.size() + 2][];
        rows[0] = GLOBAL_HEADER;
        int totalBundles = 0;
        double totalThroughput = 0;
        double totalMessageRate = 0;
        double totalLongTermMessageRate = 0;
        double maxMaxUsage = 0;
        int i = 1;
        for (final Map.Entry<String, Object> entry : loadData.entrySet()) {
            final String broker = entry.getKey();
            final Object data = entry.getValue();
            rows[i] = new Object[GLOBAL_HEADER.length];
            rows[i][0] = broker;
            int numBundles;
            double messageRate;
            double longTermMessageRate;
            double messageThroughput;
            double maxUsage;
            if (data instanceof LoadReport) {
                final LoadReport loadReport = (LoadReport) data;
                numBundles = (int) loadReport.getNumBundles();
                messageRate = loadReport.getMsgRateIn() + loadReport.getMsgRateOut();
                longTermMessageRate = loadReport.getAllocatedMsgRateIn() + loadReport.getAllocatedMsgRateOut();
                messageThroughput = (loadReport.getAllocatedBandwidthIn() + loadReport.getAllocatedBandwidthOut()) / 1024;
                final SystemResourceUsage systemResourceUsage = loadReport.getSystemResourceUsage();
                maxUsage = Math.max(Math.max(Math.max(systemResourceUsage.getCpu().percentUsage(), systemResourceUsage.getMemory().percentUsage()), Math.max(systemResourceUsage.getDirectMemory().percentUsage(), systemResourceUsage.getBandwidthIn().percentUsage())), systemResourceUsage.getBandwidthOut().percentUsage());
            } else if (data instanceof LocalBrokerData) {
                final LocalBrokerData localData = (LocalBrokerData) data;
                numBundles = localData.getNumBundles();
                messageRate = localData.getMsgRateIn() + localData.getMsgRateOut();
                final String timeAveragePath = ModularLoadManagerImpl.TIME_AVERAGE_BROKER_ZPATH + "/" + broker;
                try {
                    final TimeAverageBrokerData timeAverageData = gson.fromJson(new String(zkClient.getData(timeAveragePath, false, null)), TimeAverageBrokerData.class);
                    longTermMessageRate = timeAverageData.getLongTermMsgRateIn() + timeAverageData.getLongTermMsgRateOut();
                } catch (Exception x) {
                    throw new RuntimeException(x);
                }
                messageThroughput = (localData.getMsgThroughputIn() + localData.getMsgThroughputOut()) / 1024;
                maxUsage = localData.getMaxResourceUsage();
            } else {
                throw new AssertionError("Unreachable code");
            }
            rows[i][1] = numBundles;
            rows[i][2] = messageRate;
            rows[i][3] = messageThroughput;
            rows[i][4] = longTermMessageRate;
            rows[i][5] = maxUsage;
            totalBundles += numBundles;
            totalMessageRate += messageRate;
            totalLongTermMessageRate += longTermMessageRate;
            totalThroughput += messageThroughput;
            maxMaxUsage = Math.max(maxUsage, maxMaxUsage);
            ++i;
        }
        final int finalRow = loadData.size() + 1;
        rows[finalRow] = new Object[GLOBAL_HEADER.length];
        rows[finalRow][0] = "TOTAL";
        rows[finalRow][1] = totalBundles;
        rows[finalRow][2] = totalMessageRate;
        rows[finalRow][3] = totalLongTermMessageRate;
        rows[finalRow][4] = totalThroughput;
        rows[finalRow][5] = maxMaxUsage;
        final String table = globalTableMaker.make(rows);
        log.info("Overall Broker Data:\n{}", table);
    }
}

Example 3 with TimeAverageBrokerData

use of org.apache.pulsar.broker.TimeAverageBrokerData in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method writeBundleDataOnZooKeeper.

/**
 * As the leader broker, write bundle data aggregated from all brokers to ZooKeeper.
 */
@Override
public void writeBundleDataOnZooKeeper() {
    updateBundleData();
    // Write the bundle data to ZooKeeper.
    for (Map.Entry<String, BundleData> entry : loadData.getBundleData().entrySet()) {
        final String bundle = entry.getKey();
        final BundleData data = entry.getValue();
        try {
            final String zooKeeperPath = getBundleDataZooKeeperPath(bundle);
            createZPathIfNotExists(zkClient, zooKeeperPath);
            zkClient.setData(zooKeeperPath, data.getJsonBytes(), -1);
        } catch (Exception e) {
            log.warn("Error when writing data for bundle {} to ZooKeeper: {}", bundle, e);
        }
    }
    // Write the time average broker data to ZooKeeper.
    for (Map.Entry<String, BrokerData> entry : loadData.getBrokerData().entrySet()) {
        final String broker = entry.getKey();
        final TimeAverageBrokerData data = entry.getValue().getTimeAverageData();
        try {
            final String zooKeeperPath = TIME_AVERAGE_BROKER_ZPATH + "/" + broker;
            createZPathIfNotExists(zkClient, zooKeeperPath);
            zkClient.setData(zooKeeperPath, data.getJsonBytes(), -1);
            if (log.isDebugEnabled()) {
                log.debug("Writing zookeeper report {}", data);
            }
        } catch (Exception e) {
            log.warn("Error when writing time average broker data for {} to ZooKeeper: {}", broker, e);
        }
    }
}

Example 4 with TimeAverageBrokerData

use of org.apache.pulsar.broker.TimeAverageBrokerData in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method start.

/**
 * As any broker, start the load manager.
 *
 * @throws PulsarServerException
 *             If an unexpected error prevented the load manager from being started.
 */
@Override
public void start() throws PulsarServerException {
    try {
        // Register the brokers in zk list
        createZPathIfNotExists(zkClient, LoadManager.LOADBALANCE_BROKERS_ROOT);
        String lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" + conf.getWebServicePort();
        brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress;
        final String timeAverageZPath = TIME_AVERAGE_BROKER_ZPATH + "/" + lookupServiceAddress;
        updateLocalBrokerData();
        try {
            ZkUtils.createFullPathOptimistic(zkClient, brokerZnodePath, localData.getJsonBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
        } catch (KeeperException.NodeExistsException e) {
            long ownerZkSessionId = getBrokerZnodeOwner();
            if (ownerZkSessionId != 0 && ownerZkSessionId != zkClient.getSessionId()) {
                log.error("Broker znode - [{}] is own by different zookeeper-ssession {} ", brokerZnodePath, ownerZkSessionId);
                throw new PulsarServerException("Broker-znode owned by different zk-session " + ownerZkSessionId);
            }
            // Node may already be created by another load manager: in this case update the data.
            zkClient.setData(brokerZnodePath, localData.getJsonBytes(), -1);
        } catch (Exception e) {
            // Catching exception here to print the right error message
            log.error("Unable to create znode - [{}] for load balance on zookeeper ", brokerZnodePath, e);
            throw e;
        }
        createZPathIfNotExists(zkClient, timeAverageZPath);
        zkClient.setData(timeAverageZPath, (new TimeAverageBrokerData()).getJsonBytes(), -1);
        updateAll();
        lastBundleDataUpdate = System.currentTimeMillis();
    } catch (Exception e) {
        log.error("Unable to create znode - [{}] for load balance on zookeeper ", brokerZnodePath, e);
        throw new PulsarServerException(e);
    }
}

Example 5 with TimeAverageBrokerData

use of org.apache.pulsar.broker.TimeAverageBrokerData in project incubator-pulsar by apache.

the class ModularLoadManagerStrategyTest method initBrokerData.

private BrokerData initBrokerData() {
    LocalBrokerData localBrokerData = new LocalBrokerData();
    localBrokerData.setCpu(new ResourceUsage());
    localBrokerData.setMemory(new ResourceUsage());
    localBrokerData.setBandwidthIn(new ResourceUsage());
    localBrokerData.setBandwidthOut(new ResourceUsage());
    BrokerData brokerData = new BrokerData(localBrokerData);
    TimeAverageBrokerData timeAverageBrokerData = new TimeAverageBrokerData();
    brokerData.setTimeAverageData(timeAverageBrokerData);
    return brokerData;
}