Examples with ResourceUsage org.apache.pulsar.policies.data.loadbalancer.ResourceUsage used on opensource projects

Example 6 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testLoadShedding.

// Test that load shedding works
@Test
public void testLoadShedding() throws Exception {
    final NamespaceBundleStats stats1 = new NamespaceBundleStats();
    final NamespaceBundleStats stats2 = new NamespaceBundleStats();
    stats1.msgRateIn = 100;
    stats2.msgRateIn = 200;
    final Map<String, NamespaceBundleStats> statsMap = new ConcurrentHashMap<>();
    statsMap.put(mockBundleName(1), stats1);
    statsMap.put(mockBundleName(2), stats2);
    final LocalBrokerData localBrokerData = new LocalBrokerData();
    localBrokerData.update(new SystemResourceUsage(), statsMap);
    final Namespaces namespacesSpy1 = spy(pulsar1.getAdminClient().namespaces());
    AtomicReference<String> bundleReference = new AtomicReference<>();
    doAnswer(invocation -> {
        bundleReference.set(invocation.getArguments()[0].toString() + '/' + invocation.getArguments()[1]);
        return null;
    }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1);
    pulsar1.getConfiguration().setLoadBalancerEnabled(true);
    final LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData");
    final Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost));
    when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData);
    brokerDataMap.put(primaryHost, brokerDataSpy1);
    // Need to update all the bundle data for the shedder to see the spy.
    primaryLoadManager.onUpdate(null, null, null);
    Thread.sleep(100);
    localBrokerData.setCpu(new ResourceUsage(80, 100));
    primaryLoadManager.doLoadShedding();
    // 80% is below overload threshold: verify nothing is unloaded.
    verify(namespacesSpy1, Mockito.times(0)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    localBrokerData.getCpu().usage = 90;
    primaryLoadManager.doLoadShedding();
    // Most expensive bundle will be unloaded.
    verify(namespacesSpy1, Mockito.times(1)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    assertEquals(bundleReference.get(), mockBundleName(2));
    primaryLoadManager.doLoadShedding();
    // Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be
    // unloaded, but this is not the case due to the spy's behavior).
    verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    assertEquals(bundleReference.get(), mockBundleName(1));
    primaryLoadManager.doLoadShedding();
    // Now both are in grace period: neither should be unloaded.
    verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
}

Example 7 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testNeedBrokerDataUpdate.

// Test that ModularLoadManagerImpl will determine that writing local data to ZooKeeper is necessary if certain
// metrics change by a percentage threshold.
@Test
public void testNeedBrokerDataUpdate() throws Exception {
    final LocalBrokerData lastData = new LocalBrokerData();
    final LocalBrokerData currentData = new LocalBrokerData();
    final ServiceConfiguration conf = pulsar1.getConfiguration();
    // Set this manually in case the default changes.
    conf.setLoadBalancerReportUpdateThresholdPercentage(5);
    // Easier to test using an uninitialized ModularLoadManagerImpl.
    final ModularLoadManagerImpl loadManager = new ModularLoadManagerImpl();
    setField(loadManager, "lastData", lastData);
    setField(loadManager, "localData", currentData);
    setField(loadManager, "conf", conf);
    Supplier<Boolean> needUpdate = () -> {
        try {
            return (Boolean) invokeSimpleMethod(loadManager, "needBrokerDataUpdate");
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    };
    lastData.setMsgRateIn(100);
    currentData.setMsgRateIn(104);
    // 4% difference: shouldn't trigger an update.
    assert (!needUpdate.get());
    currentData.setMsgRateIn(105.1);
    // 5% difference: should trigger an update (exactly 5% is flaky due to precision).
    assert (needUpdate.get());
    // Do similar tests for lower values.
    currentData.setMsgRateIn(94);
    assert (needUpdate.get());
    currentData.setMsgRateIn(95.1);
    assert (!needUpdate.get());
    // 0 to non-zero should always trigger an update.
    lastData.setMsgRateIn(0);
    currentData.setMsgRateIn(1e-8);
    assert (needUpdate.get());
    // non-zero to zero should trigger an update as long as the threshold is less than 100.
    lastData.setMsgRateIn(1e-8);
    currentData.setMsgRateIn(0);
    assert (needUpdate.get());
    // zero to zero should never trigger an update.
    currentData.setMsgRateIn(0);
    lastData.setMsgRateIn(0);
    assert (!needUpdate.get());
    // Minimally test other absolute values to ensure they are included.
    lastData.getCpu().usage = 100;
    lastData.getCpu().limit = 1000;
    currentData.getCpu().usage = 106;
    currentData.getCpu().limit = 1000;
    assert (!needUpdate.get());
    // Minimally test other absolute values to ensure they are included.
    lastData.getCpu().usage = 100;
    lastData.getCpu().limit = 1000;
    currentData.getCpu().usage = 206;
    currentData.getCpu().limit = 1000;
    assert (needUpdate.get());
    lastData.setCpu(new ResourceUsage());
    currentData.setCpu(new ResourceUsage());
    lastData.setMsgThroughputIn(100);
    currentData.setMsgThroughputIn(106);
    assert (needUpdate.get());
    currentData.setMsgThroughputIn(100);
    lastData.setNumBundles(100);
    currentData.setNumBundles(106);
    assert (needUpdate.get());
    currentData.setNumBundles(100);
    assert (!needUpdate.get());
}

Example 8 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class ModularLoadManagerStrategyTest method testLeastLongTermMessageRate.

// Test that least long term message rate works correctly.
@Test
public void testLeastLongTermMessageRate() {
    BundleData bundleData = new BundleData();
    BrokerData brokerData1 = initBrokerData();
    BrokerData brokerData2 = initBrokerData();
    BrokerData brokerData3 = initBrokerData();
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(100);
    brokerData2.getTimeAverageData().setLongTermMsgRateIn(200);
    brokerData3.getTimeAverageData().setLongTermMsgRateIn(300);
    LoadData loadData = new LoadData();
    Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    brokerDataMap.put("1", brokerData1);
    brokerDataMap.put("2", brokerData2);
    brokerDataMap.put("3", brokerData3);
    ServiceConfiguration conf = new ServiceConfiguration();
    ModularLoadManagerStrategy strategy = new LeastLongTermMessageRate(conf);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("1"));
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(400);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2"));
    brokerData2.getLocalData().setCpu(new ResourceUsage(90, 100));
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("3"));
}

Example 9 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testPrimarySecondary.

@Test(enabled = false)
public void testPrimarySecondary() throws Exception {
    createNamespacePolicies(pulsar1);
    LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
    ZooKeeperChildrenCache zooKeeperChildrenCache = mock(ZooKeeperChildrenCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod2-broker7.messaging.use.example.com:8080", "prod2-broker8.messaging.use.example.com:8080", "prod2-broker9.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(zooKeeperChildrenCache.get()).thenReturn(activeBrokers);
    when(zooKeeperChildrenCache.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsar1);
    LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsar2);
    log.info("lzk are {} 2: {}", originalLZK1.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT), originalLZK2.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    zkCacheField.set(pulsar1, mockCache);
    LocalZooKeeperCache newZk = (LocalZooKeeperCache) pulsar1.getLocalZkCache();
    log.info("lzk mocked are {}", newZk.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    ZooKeeperChildrenCache availableActiveBrokers = new ZooKeeperChildrenCache(pulsar1.getLocalZkCache(), SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT);
    log.info("lzk mocked active brokers are {}", availableActiveBrokers.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
    assertEquals(found.getResourceId(), ru1.getResourceId());
    zkCacheField.set(pulsar1, originalLZK1);
}

Example 10 with ResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.ResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testLoadReportParsing.

@Test
public void testLoadReportParsing() throws Exception {
    ObjectMapper mapper = ObjectMapperFactory.create();
    org.apache.pulsar.policies.data.loadbalancer.LoadReport reportData = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    reportData.setName("b1");
    SystemResourceUsage resource = new SystemResourceUsage();
    ResourceUsage resourceUsage = new ResourceUsage();
    resource.setBandwidthIn(resourceUsage);
    resource.setBandwidthOut(resourceUsage);
    resource.setMemory(resourceUsage);
    resource.setCpu(resourceUsage);
    reportData.setSystemResourceUsage(resource);
    String loadReportJson = mapper.writeValueAsString(reportData);
    LoadReport loadReport = PulsarLoadReportImpl.parse(loadReportJson);
    assertEquals(loadReport.getResourceUnitDescription().getResourceUsage().get("bandwidthIn").compareTo(resourceUsage), 0);
}