Search in sources :

Example 6 with ResourceAvailabilityRanker

use of org.apache.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker in project incubator-pulsar by apache.

the class LoadBalancerTest method testLoadbalanceDistributionAmongEquallyLoaded.

/*
     * Simple Test, creates three Resource Units (brokers) with equal load and expects that out of 1005 namespaces it
     * should be divided fairly equally with about 10% of variation
     *
     */
@Test(enabled = false)
public void testLoadbalanceDistributionAmongEquallyLoaded() throws Exception {
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    zkCacheField.set(pulsarServices[0], mockCache);
    long memoryMB = 4096;
    long cpuPercent = 45;
    long bInMbps = 350;
    long bOutMbps = 180;
    long threads = 10;
    // TODO move to its own test
    PulsarResourceDescription rd = createResourceDescription(memoryMB, cpuPercent, bInMbps, bOutMbps, threads);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    for (String broker : activeBrokers) {
        ResourceUnit ru = new SimpleResourceUnit(broker, rd);
        rus.add(ru);
    }
    TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
    LoadRanker ranker = new ResourceAvailabilityRanker();
    sortedRankingsInstance.put(ranker.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
    ar.set(sortedRankingsInstance);
    sortedRankings.set(loadManager, ar);
}
Also used : PulsarResourceDescription(org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription) Set(java.util.Set) HashSet(java.util.HashSet) ResourceAvailabilityRanker(org.apache.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker) AtomicReference(java.util.concurrent.atomic.AtomicReference) TreeMap(java.util.TreeMap) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) Field(java.lang.reflect.Field) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) LocalZooKeeperCache(org.apache.pulsar.zookeeper.LocalZooKeeperCache) ZooKeeperCache(org.apache.pulsar.zookeeper.ZooKeeperCache) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 7 with ResourceAvailabilityRanker

use of org.apache.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testDoLoadShedding.

@Test(enabled = true)
public void testDoLoadShedding() throws Exception {
    LoadManager loadManager = spy(new SimpleLoadManagerImpl(pulsar1));
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://pulsar-broker1.com:8080", rd);
    ResourceUnit ru2 = new SimpleResourceUnit("http://pulsar-broker2.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    rus.add(ru2);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    // inject the load report and rankings
    SystemResourceUsage systemResource = new SystemResourceUsage();
    systemResource.setBandwidthIn(new ResourceUsage(90, 100));
    Map<String, NamespaceBundleStats> stats = Maps.newHashMap();
    NamespaceBundleStats nsb1 = new NamespaceBundleStats();
    nsb1.msgRateOut = 10000;
    NamespaceBundleStats nsb2 = new NamespaceBundleStats();
    nsb2.msgRateOut = 10000;
    stats.put("property/cluster/namespace1/0x00000000_0xFFFFFFFF", nsb1);
    stats.put("property/cluster/namespace2/0x00000000_0xFFFFFFFF", nsb2);
    Map<ResourceUnit, org.apache.pulsar.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport1 = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport1.setSystemResourceUsage(systemResource);
    loadReport1.setBundleStats(stats);
    org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport2 = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
    loadReport2.setSystemResourceUsage(new SystemResourceUsage());
    loadReport2.setBundleStats(stats);
    loadReports.put(ru1, loadReport1);
    loadReports.put(ru2, loadReport2);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ((SimpleLoadManagerImpl) loadManager).doLoadShedding();
    verify(loadManager, atLeastOnce()).doLoadShedding();
}
Also used : PulsarResourceDescription(org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription) HashMap(java.util.HashMap) ResourceAvailabilityRanker(org.apache.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker) Field(java.lang.reflect.Field) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) HashSet(java.util.HashSet) SystemResourceUsage(org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage) ResourceUsage(org.apache.pulsar.policies.data.loadbalancer.ResourceUsage) SystemResourceUsage(org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage) AtomicReference(java.util.concurrent.atomic.AtomicReference) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.testng.annotations.Test)

Example 8 with ResourceAvailabilityRanker

use of org.apache.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testBasicBrokerSelection.

@Test(enabled = true)
public void testBasicBrokerSelection() throws Exception {
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    Optional<ResourceUnit> res = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10"));
    // broker is not active so found should be null
    assertEquals(res, Optional.empty(), "found a broker when expected none to be found");
}
Also used : PulsarResourceDescription(org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription) SystemResourceUsage(org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage) ResourceUsage(org.apache.pulsar.policies.data.loadbalancer.ResourceUsage) ResourceAvailabilityRanker(org.apache.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker) AtomicReference(java.util.concurrent.atomic.AtomicReference) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) Field(java.lang.reflect.Field) SimpleResourceUnit(org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit) SimpleLoadManagerImpl(org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) Map(java.util.Map) HashMap(java.util.HashMap) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Aggregations

HashSet (java.util.HashSet)8 AtomicReference (java.util.concurrent.atomic.AtomicReference)8 PulsarResourceDescription (org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription)8 ResourceAvailabilityRanker (org.apache.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker)8 SimpleLoadManagerImpl (org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl)8 SimpleResourceUnit (org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit)8 Test (org.testng.annotations.Test)8 Field (java.lang.reflect.Field)7 HashMap (java.util.HashMap)7 Map (java.util.Map)6 ResourceUsage (org.apache.pulsar.policies.data.loadbalancer.ResourceUsage)5 SystemResourceUsage (org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage)5 LocalZooKeeperCache (org.apache.pulsar.zookeeper.LocalZooKeeperCache)5 TreeMap (java.util.TreeMap)4 Set (java.util.Set)3 ZooKeeperCache (org.apache.pulsar.zookeeper.ZooKeeperCache)3 ResourceQuota (org.apache.pulsar.common.policies.data.ResourceQuota)1 NamespaceBundleStats (org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats)1 ResourceUnitRanking (org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking)1 ZooKeeperChildrenCache (org.apache.pulsar.zookeeper.ZooKeeperChildrenCache)1