Examples with BrokerData org.apache.pulsar.broker.BrokerData used on opensource projects

Example 6 with BrokerData

use of org.apache.pulsar.broker.BrokerData in project incubator-pulsar by apache.

the class BrokerVersionFilterTest method initLoadData.

private LoadData initLoadData() {
    LocalBrokerData broker1Data = new LocalBrokerData();
    broker1Data.setBrokerVersionString("1.1.0-SNAPSHOT");
    LocalBrokerData broker2Data = new LocalBrokerData();
    broker2Data.setBrokerVersionString("1.1.0-SNAPSHOT");
    LocalBrokerData broker3Data = new LocalBrokerData();
    broker3Data.setBrokerVersionString("1.2.0-SNAPSHOT");
    LoadData loadData = new LoadData();
    loadData.getBrokerData().put("broker1", new BrokerData(broker1Data));
    loadData.getBrokerData().put("broker2", new BrokerData(broker2Data));
    loadData.getBrokerData().put("broker3", new BrokerData(broker3Data));
    return loadData;
}

Example 7 with BrokerData

use of org.apache.pulsar.broker.BrokerData in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testLoadShedding.

// Test that load shedding works
@Test
public void testLoadShedding() throws Exception {
    final NamespaceBundleStats stats1 = new NamespaceBundleStats();
    final NamespaceBundleStats stats2 = new NamespaceBundleStats();
    stats1.msgRateIn = 100;
    stats2.msgRateIn = 200;
    final Map<String, NamespaceBundleStats> statsMap = new ConcurrentHashMap<>();
    statsMap.put(mockBundleName(1), stats1);
    statsMap.put(mockBundleName(2), stats2);
    final LocalBrokerData localBrokerData = new LocalBrokerData();
    localBrokerData.update(new SystemResourceUsage(), statsMap);
    final Namespaces namespacesSpy1 = spy(pulsar1.getAdminClient().namespaces());
    AtomicReference<String> bundleReference = new AtomicReference<>();
    doAnswer(invocation -> {
        bundleReference.set(invocation.getArguments()[0].toString() + '/' + invocation.getArguments()[1]);
        return null;
    }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1);
    pulsar1.getConfiguration().setLoadBalancerEnabled(true);
    final LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData");
    final Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost));
    when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData);
    brokerDataMap.put(primaryHost, brokerDataSpy1);
    // Need to update all the bundle data for the shedder to see the spy.
    primaryLoadManager.onUpdate(null, null, null);
    Thread.sleep(100);
    localBrokerData.setCpu(new ResourceUsage(80, 100));
    primaryLoadManager.doLoadShedding();
    // 80% is below overload threshold: verify nothing is unloaded.
    verify(namespacesSpy1, Mockito.times(0)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    localBrokerData.getCpu().usage = 90;
    primaryLoadManager.doLoadShedding();
    // Most expensive bundle will be unloaded.
    verify(namespacesSpy1, Mockito.times(1)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    assertEquals(bundleReference.get(), mockBundleName(2));
    primaryLoadManager.doLoadShedding();
    // Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be
    // unloaded, but this is not the case due to the spy's behavior).
    verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    assertEquals(bundleReference.get(), mockBundleName(1));
    primaryLoadManager.doLoadShedding();
    // Now both are in grace period: neither should be unloaded.
    verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
}

Example 8 with BrokerData

use of org.apache.pulsar.broker.BrokerData in project incubator-pulsar by apache.

the class ModularLoadManagerStrategyTest method testLeastLongTermMessageRate.

// Test that least long term message rate works correctly.
@Test
public void testLeastLongTermMessageRate() {
    BundleData bundleData = new BundleData();
    BrokerData brokerData1 = initBrokerData();
    BrokerData brokerData2 = initBrokerData();
    BrokerData brokerData3 = initBrokerData();
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(100);
    brokerData2.getTimeAverageData().setLongTermMsgRateIn(200);
    brokerData3.getTimeAverageData().setLongTermMsgRateIn(300);
    LoadData loadData = new LoadData();
    Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    brokerDataMap.put("1", brokerData1);
    brokerDataMap.put("2", brokerData2);
    brokerDataMap.put("3", brokerData3);
    ServiceConfiguration conf = new ServiceConfiguration();
    ModularLoadManagerStrategy strategy = new LeastLongTermMessageRate(conf);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("1"));
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(400);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2"));
    brokerData2.getLocalData().setCpu(new ResourceUsage(90, 100));
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("3"));
}

Example 9 with BrokerData

use of org.apache.pulsar.broker.BrokerData in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method writeBundleDataOnZooKeeper.

/**
 * As the leader broker, write bundle data aggregated from all brokers to ZooKeeper.
 */
@Override
public void writeBundleDataOnZooKeeper() {
    updateBundleData();
    // Write the bundle data to ZooKeeper.
    for (Map.Entry<String, BundleData> entry : loadData.getBundleData().entrySet()) {
        final String bundle = entry.getKey();
        final BundleData data = entry.getValue();
        try {
            final String zooKeeperPath = getBundleDataZooKeeperPath(bundle);
            createZPathIfNotExists(zkClient, zooKeeperPath);
            zkClient.setData(zooKeeperPath, data.getJsonBytes(), -1);
        } catch (Exception e) {
            log.warn("Error when writing data for bundle {} to ZooKeeper: {}", bundle, e);
        }
    }
    // Write the time average broker data to ZooKeeper.
    for (Map.Entry<String, BrokerData> entry : loadData.getBrokerData().entrySet()) {
        final String broker = entry.getKey();
        final TimeAverageBrokerData data = entry.getValue().getTimeAverageData();
        try {
            final String zooKeeperPath = TIME_AVERAGE_BROKER_ZPATH + "/" + broker;
            createZPathIfNotExists(zkClient, zooKeeperPath);
            zkClient.setData(zooKeeperPath, data.getJsonBytes(), -1);
            if (log.isDebugEnabled()) {
                log.debug("Writing zookeeper report {}", data);
            }
        } catch (Exception e) {
            log.warn("Error when writing time average broker data for {} to ZooKeeper: {}", broker, e);
        }
    }
}

Example 10 with BrokerData

use of org.apache.pulsar.broker.BrokerData in project incubator-pulsar by apache.

the class BrokerVersionFilter method getLatestVersionNumber.

/**
 * Get the most recent broker version number from the load reports of all the running brokers. The version
 * number is from the build artifact in the pom and got added to the package when it was built by Maven
 *
 * @param brokers
 *            The brokers to choose the latest version string from.
 * @param loadData
 *            The load data from the leader broker (contains the load reports which in turn contain the version string).
 * @return The most recent broker version
 * @throws BrokerFilterBadVersionException
 *            If the most recent version is undefined (e.g., a bad broker version was encountered or a broker
 *            does not have a version string in its load report.
 */
public Version getLatestVersionNumber(Set<String> brokers, LoadData loadData) throws BrokerFilterBadVersionException {
    if (null == brokers) {
        throw new BrokerFilterBadVersionException("Unable to determine latest version since broker set was null");
    }
    if (brokers.size() == 0) {
        throw new BrokerFilterBadVersionException("Unable to determine latest version since broker set was empty");
    }
    if (null == loadData) {
        throw new BrokerFilterBadVersionException("Unable to determine latest version since loadData was null");
    }
    Version latestVersion = null;
    for (String broker : brokers) {
        BrokerData data = loadData.getBrokerData().get(broker);
        if (null == data) {
            LOG.warn("No broker data for broker [{}]; disabling PreferLaterVersions feature", broker);
            // trigger the ModularLoadManager to reset all the brokers to the original set
            throw new BrokerFilterBadVersionException("No broker data for broker \"" + broker + "\"");
        }
        String brokerVersion = data.getLocalData().getBrokerVersionString();
        if (null == brokerVersion || brokerVersion.length() == 0) {
            LOG.warn("No version string in load report for broker [{}]; disabling PreferLaterVersions feature", broker);
            // trigger the ModularLoadManager to reset all the brokers to the original set
            throw new BrokerFilterBadVersionException("No version string in load report for broker \"" + broker + "\"");
        }
        Version brokerVersionVersion = null;
        try {
            brokerVersionVersion = Version.valueOf(brokerVersion);
        } catch (Exception x) {
            LOG.warn("Invalid version string in load report for broker [{}]: [{}]; disabling PreferLaterVersions feature", broker, brokerVersion);
            // trigger the ModularLoadManager to reset all the brokers to the original set
            throw new BrokerFilterBadVersionException("Invalid version string in load report for broker \"" + broker + "\": \"" + brokerVersion + "\")");
        }
        if (null == latestVersion) {
            latestVersion = brokerVersionVersion;
        } else if (Version.BUILD_AWARE_ORDER.compare(latestVersion, brokerVersionVersion) < 0) {
            latestVersion = brokerVersionVersion;
        }
    }
    if (null == latestVersion) {
        throw new BrokerFilterBadVersionException("Unable to determine latest broker version");
    }
    return latestVersion;
}