Examples with SystemResourceUsage org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage used on opensource projects

Example 11 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class LoadBalancerTest method testBrokerRanking.

/*
     * Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
     * bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - non-bundles assigned so all
     * idle resources are avaiable for new bundle Check the broker rankings are the load percentage of each broker.
     */
@Test
public void testBrokerRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(0, 1024000));
        sru.setBandwidthOut(new ResourceUsage(0, 1024000));
        sru.setMemory(new ResourceUsage(1024, 2048 * (i + 1)));
        sru.setCpu(new ResourceUsage(60, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait load ranking be triggered
    Thread.sleep(5000);
    // check the ranking result
    for (int i = 0; i < BROKER_COUNT; i++) {
        AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
        printSortedRanking(sortedRanking);
        // brokers' ranking would be:
        // 50 --> broker 0 ( 1024 / 2048 )
        // 25 --> broker 1 ( 1024 / 4096 )
        // 16 --> broker 2 ( 1024 / 6144 )
        // 15 --> broker 3 ( 60 / 400 )
        // 15 --> broker 4 ( 60 / 400 )
        assertEquals(sortedRanking.get().size(), 4);
        assertEquals(sortedRanking.get().get(50L).size(), 1);
        assertEquals(sortedRanking.get().get(25L).size(), 1);
        assertEquals(sortedRanking.get().get(16L).size(), 1);
        assertEquals(sortedRanking.get().get(15L).size(), 2);
    }
}

Example 12 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class LoadBalancerTest method testNamespaceBundleAutoSplit.

/**
 * Test the namespace bundle auto-split
 */
@Test
public void testNamespaceBundleAutoSplit() throws Exception {
    int maxBundles = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxTopics = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    int maxSessions = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxMsgRate = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBandwidth = pulsarServices[0].getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * 1048576;
    pulsarServices[0].getConfiguration().setLoadBalancerAutoBundleSplitEnabled(true);
    // create namespaces
    for (int i = 1; i <= 10; i++) {
        int numBundles = (i == 10) ? maxBundles : 2;
        createNamespace(pulsarServices[0], String.format("pulsar/use/primary-ns-%02d", i), numBundles);
    }
    // fake Namespaces Admin
    NamespacesImpl namespaceAdmin = mock(NamespacesImpl.class);
    setObjectField(PulsarAdmin.class, pulsarServices[0].getAdminClient(), "namespaces", namespaceAdmin);
    // create load report
    // namespace 01~09 need to be split
    // namespace 08~10 don't need or cannot be split
    LoadReport lr = new LoadReport();
    lr.setName(lookupAddresses[0]);
    lr.setSystemResourceUsage(new SystemResourceUsage());
    Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
    bundleStats.put("pulsar/use/primary-ns-01/0x00000000_0x80000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-02/0x00000000_0x80000000", newBundleStats(2, maxSessions + 1, 0, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-03/0x00000000_0x80000000", newBundleStats(2, 0, maxSessions + 1, 0, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-04/0x00000000_0x80000000", newBundleStats(2, 0, 0, maxMsgRate + 1, 0, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-05/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, maxMsgRate + 1, 0, 0));
    bundleStats.put("pulsar/use/primary-ns-06/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, maxBandwidth + 1, 0));
    bundleStats.put("pulsar/use/primary-ns-07/0x00000000_0x80000000", newBundleStats(2, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-08/0x00000000_0x80000000", newBundleStats(maxTopics - 1, maxSessions - 1, 1, maxMsgRate - 1, 1, maxBandwidth - 1, 1));
    bundleStats.put("pulsar/use/primary-ns-09/0x00000000_0x80000000", newBundleStats(1, 0, 0, 0, 0, 0, maxBandwidth + 1));
    bundleStats.put("pulsar/use/primary-ns-10/0x00000000_0x02000000", newBundleStats(maxTopics + 1, 0, 0, 0, 0, 0, 0));
    lr.setBundleStats(bundleStats);
    setObjectField(SimpleLoadManagerImpl.class, pulsarServices[0].getLoadManager().get(), "lastLoadReport", lr);
    String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[0]);
    String loadReportJson = ObjectMapperFactory.getThreadLocal().writeValueAsString(lr);
    bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    // sleep to wait load ranking be triggered and trigger bundle split
    Thread.sleep(5000);
    pulsarServices[0].getLoadManager().get().doNamespaceBundleSplit();
    boolean isAutoUnooadSplitBundleEnabled = pulsarServices[0].getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled();
    // verify bundles are split
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-01", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-02", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-03", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-04", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-05", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-06", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, times(1)).splitNamespaceBundle("pulsar/use/primary-ns-07", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-08", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-09", "0x00000000_0x80000000", isAutoUnooadSplitBundleEnabled);
    verify(namespaceAdmin, never()).splitNamespaceBundle("pulsar/use/primary-ns-10", "0x00000000_0x02000000", isAutoUnooadSplitBundleEnabled);
}

Example 13 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testLoadShedding.

// Test that load shedding works
@Test
public void testLoadShedding() throws Exception {
    final NamespaceBundleStats stats1 = new NamespaceBundleStats();
    final NamespaceBundleStats stats2 = new NamespaceBundleStats();
    stats1.msgRateIn = 100;
    stats2.msgRateIn = 200;
    final Map<String, NamespaceBundleStats> statsMap = new ConcurrentHashMap<>();
    statsMap.put(mockBundleName(1), stats1);
    statsMap.put(mockBundleName(2), stats2);
    final LocalBrokerData localBrokerData = new LocalBrokerData();
    localBrokerData.update(new SystemResourceUsage(), statsMap);
    final Namespaces namespacesSpy1 = spy(pulsar1.getAdminClient().namespaces());
    AtomicReference<String> bundleReference = new AtomicReference<>();
    doAnswer(invocation -> {
        bundleReference.set(invocation.getArguments()[0].toString() + '/' + invocation.getArguments()[1]);
        return null;
    }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1);
    pulsar1.getConfiguration().setLoadBalancerEnabled(true);
    final LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData");
    final Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost));
    when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData);
    brokerDataMap.put(primaryHost, brokerDataSpy1);
    // Need to update all the bundle data for the shedder to see the spy.
    primaryLoadManager.onUpdate(null, null, null);
    Thread.sleep(100);
    localBrokerData.setCpu(new ResourceUsage(80, 100));
    primaryLoadManager.doLoadShedding();
    // 80% is below overload threshold: verify nothing is unloaded.
    verify(namespacesSpy1, Mockito.times(0)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    localBrokerData.getCpu().usage = 90;
    primaryLoadManager.doLoadShedding();
    // Most expensive bundle will be unloaded.
    verify(namespacesSpy1, Mockito.times(1)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    assertEquals(bundleReference.get(), mockBundleName(2));
    primaryLoadManager.doLoadShedding();
    // Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be
    // unloaded, but this is not the case due to the spy's behavior).
    verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    assertEquals(bundleReference.get(), mockBundleName(1));
    primaryLoadManager.doLoadShedding();
    // Now both are in grace period: neither should be unloaded.
    verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
}

Example 14 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testLoadReportParsing.

@Test
public void testLoadReportParsing() throws Exception {
    ObjectMapper mapper = ObjectMapperFactory.create();
    org.apache.pulsar.policies.data.loadbalancer.LoadReport reportData = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    reportData.setName("b1");
    SystemResourceUsage resource = new SystemResourceUsage();
    ResourceUsage resourceUsage = new ResourceUsage();
    resource.setBandwidthIn(resourceUsage);
    resource.setBandwidthOut(resourceUsage);
    resource.setMemory(resourceUsage);
    resource.setCpu(resourceUsage);
    reportData.setSystemResourceUsage(resource);
    String loadReportJson = mapper.writeValueAsString(reportData);
    LoadReport loadReport = PulsarLoadReportImpl.parse(loadReportJson);
    assertEquals(loadReport.getResourceUnitDescription().getResourceUsage().get("bandwidthIn").compareTo(resourceUsage), 0);
}

Example 15 with SystemResourceUsage

use of org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testPrimary.

@Test(enabled = true)
public void testPrimary() throws Exception {
    createNamespacePolicies(pulsar1);
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort(), rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    // inject the load report and rankings
    Map<ResourceUnit, org.apache.pulsar.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport.setSystemResourceUsage(new SystemResourceUsage());
    loadReports.put(ru1, loadReport);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), new HashSet<String>(), new ResourceQuota(), new HashSet<String>(), new ResourceQuota());
    Map<ResourceUnit, ResourceUnitRanking> rankings = new HashMap<>();
    rankings.put(ru1, ranking);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "resourceUnitRankings", rankings);
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "sortedRankings", sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
    // broker is not active so found should be null
    assertNotEquals(found, null, "did not find a broker when expected one to be found");
}