Search in sources :

Example 1 with SimpleLoadManagerImpl

use of com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project pulsar by yahoo.

the class LoadBalancerTest method testLoadBalanceDistributionAmongUnequallyLoaded.

@Test(enabled = false)
void testLoadBalanceDistributionAmongUnequallyLoaded() throws Exception {
    long memoryMB = 4096;
    long cpuPercent = 25;
    long bInMbps = 256;
    long bOutMbps = 256;
    long threads = 25;
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    zkCacheField.set(pulsarServices[0], mockCache);
    int totalAvailabilityWeight = 0;
    TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
    for (int i = 1; i <= 3; i++) {
        PulsarResourceDescription rd = createResourceDescription(memoryMB * i, cpuPercent * i, bInMbps * i, bOutMbps * 2, threads * i);
        ResourceUnit ru1 = new SimpleResourceUnit(String.format("http://prod1-broker%d.messaging.use.example.com:8080", i), rd);
        LoadRanker ranker = new ResourceAvailabilityRanker();
        long rank = ranker.getRank(rd);
        if (sortedRankingsInstance.containsKey(rank)) {
            // get the object set and put the rd in it
            sortedRankingsInstance.get(rank).add(ru1);
        } else {
            Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
            rus.add(ru1);
            sortedRankingsInstance.put(rank, rus);
        }
        totalAvailabilityWeight += rank;
    }
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
    ar.set(sortedRankingsInstance);
    sortedRankings.set(loadManager, ar);
    int totalNamespaces = 1000;
    Map<String, Integer> namespaceOwner = new HashMap<String, Integer>();
    for (int i = 0; i < totalNamespaces; i++) {
        ResourceUnit found = loadManager.getLeastLoaded(DestinationName.get("persistent://pulsar/use/primary-ns/topic-" + i));
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 0);
        }
    }
    for (Map.Entry<Long, Set<ResourceUnit>> entry : sortedRankingsInstance.entrySet()) {
        int selectionProbability = (int) (((double) entry.getKey() / totalAvailabilityWeight) * 100);
        int idealExpectedOwned = selectionProbability * (int) ((double) totalNamespaces / 100);
        int expectedOwnedLowerBound = idealExpectedOwned - idealExpectedOwned / 10;
        int expectedOwnedUpperBound = idealExpectedOwned + idealExpectedOwned / 10;
        for (ResourceUnit ru : entry.getValue()) {
            assertTrue(namespaceOwner.containsKey(ru.getResourceId()));
            int ownedNamespaces = namespaceOwner.get(ru.getResourceId());
            assertTrue(ownedNamespaces > expectedOwnedLowerBound || ownedNamespaces < expectedOwnedUpperBound);
        }
    }
}
Also used : PulsarResourceDescription(com.yahoo.pulsar.broker.loadbalance.impl.PulsarResourceDescription) Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) ResourceAvailabilityRanker(com.yahoo.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker) Field(java.lang.reflect.Field) SimpleResourceUnit(com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit) SimpleLoadManagerImpl(com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) LocalZooKeeperCache(com.yahoo.pulsar.zookeeper.LocalZooKeeperCache) ZooKeeperCache(com.yahoo.pulsar.zookeeper.ZooKeeperCache) HashSet(java.util.HashSet) AtomicReference(java.util.concurrent.atomic.AtomicReference) TreeMap(java.util.TreeMap) SimpleResourceUnit(com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit) Map(java.util.Map) HashMap(java.util.HashMap) TreeMap(java.util.TreeMap) Test(org.testng.annotations.Test)

Example 2 with SimpleLoadManagerImpl

use of com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project pulsar by yahoo.

the class BrokerStats method getBrokerResourceAvailability.

@GET
@Path("/broker-resource-availability/{property}/{cluster}/{namespace}")
@ApiOperation(value = "Broker availability report", notes = "This API gives the current broker availability in percent, each resource percentage usage is calculated and then" + "sum of all of the resource usage percent is called broker-resource-availability" + "<br/><br/>THIS API IS ONLY FOR USE BY TESTING FOR CONFIRMING NAMESPACE ALLOCATION ALGORITHM", response = ResourceUnit.class, responseContainer = "Map")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") })
public Map<Long, Collection<ResourceUnit>> getBrokerResourceAvailability(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace) throws Exception {
    try {
        NamespaceName ns = new NamespaceName(property, cluster, namespace);
        SimpleLoadManagerImpl lm = (SimpleLoadManagerImpl) (pulsar().getLoadManager());
        return lm.getResourceAvailabilityFor(ns).asMap();
    } catch (Exception e) {
        log.error("Unable to get Resource Availability - [{}]", e);
        throw new RestException(e);
    }
}
Also used : NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) SimpleLoadManagerImpl(com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) RestException(com.yahoo.pulsar.broker.web.RestException) RestException(com.yahoo.pulsar.broker.web.RestException) WebApplicationException(javax.ws.rs.WebApplicationException) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 3 with SimpleLoadManagerImpl

use of com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project pulsar by yahoo.

the class LoadBalancerTest method testLoadbalanceDistributionAmongEquallyLoaded.

/*
     * Simple Test, creates three Resource Units (brokers) with equal load and expects that out of 1005 namespaces it
     * should be divided fairly equally with about 10% of variation
     *
     */
@Test(enabled = false)
public void testLoadbalanceDistributionAmongEquallyLoaded() throws Exception {
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    zkCacheField.set(pulsarServices[0], mockCache);
    long memoryMB = 4096;
    long cpuPercent = 45;
    long bInMbps = 350;
    long bOutMbps = 180;
    long threads = 10;
    // TODO move to its own test
    PulsarResourceDescription rd = createResourceDescription(memoryMB, cpuPercent, bInMbps, bOutMbps, threads);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    for (String broker : activeBrokers) {
        ResourceUnit ru = new SimpleResourceUnit(broker, rd);
        rus.add(ru);
    }
    TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
    LoadRanker ranker = new ResourceAvailabilityRanker();
    sortedRankingsInstance.put(ranker.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
    ar.set(sortedRankingsInstance);
    sortedRankings.set(loadManager, ar);
}
Also used : PulsarResourceDescription(com.yahoo.pulsar.broker.loadbalance.impl.PulsarResourceDescription) Set(java.util.Set) HashSet(java.util.HashSet) ResourceAvailabilityRanker(com.yahoo.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker) AtomicReference(java.util.concurrent.atomic.AtomicReference) TreeMap(java.util.TreeMap) SimpleResourceUnit(com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit) Field(java.lang.reflect.Field) SimpleResourceUnit(com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit) SimpleLoadManagerImpl(com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) LocalZooKeeperCache(com.yahoo.pulsar.zookeeper.LocalZooKeeperCache) ZooKeeperCache(com.yahoo.pulsar.zookeeper.ZooKeeperCache) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 4 with SimpleLoadManagerImpl

use of com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project pulsar by yahoo.

the class LoadBalancerTest method testGetLeastLoadedBasic.

@Test(enabled = false)
public void testGetLeastLoadedBasic() throws Exception {
    LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[0]);
    LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[1]);
    zkCacheField.set(pulsarServices[0], mockCache);
    zkCacheField.set(pulsarServices[1], mockCache);
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    // TODO move to its own test
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod1-broker1.messaging.use.example.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(new NamespaceName("pulsar/use/primary-ns.10"));
    assertEquals("http://prod1-broker1.messaging.use.example.com:8080", found.getResourceId());
    zkCacheField.set(pulsarServices[0], originalLZK1);
    zkCacheField.set(pulsarServices[1], originalLZK2);
}
Also used : PulsarResourceDescription(com.yahoo.pulsar.broker.loadbalance.impl.PulsarResourceDescription) LocalZooKeeperCache(com.yahoo.pulsar.zookeeper.LocalZooKeeperCache) SystemResourceUsage(com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage) ResourceUsage(com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage) ResourceAvailabilityRanker(com.yahoo.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker) AtomicReference(java.util.concurrent.atomic.AtomicReference) SimpleResourceUnit(com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit) Field(java.lang.reflect.Field) NamespaceName(com.yahoo.pulsar.common.naming.NamespaceName) SimpleResourceUnit(com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit) SimpleLoadManagerImpl(com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) Map(java.util.Map) HashMap(java.util.HashMap) TreeMap(java.util.TreeMap) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 5 with SimpleLoadManagerImpl

use of com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl in project pulsar by yahoo.

the class LoadBalancerTest method testLoadBalanceDiscardingInactiveBrokersInSelection.

/**
     * This test puts few brokers in the ranking that load balance uses but does not put few brokers in the mock
     * zookeeper cache, if the broker is not in the zk cache, it's considered inactive.
     *
     * We should not see any of these inactive brokers assigned any namespace.
     */
@Test(enabled = false)
void testLoadBalanceDiscardingInactiveBrokersInSelection() throws Exception {
    long memoryMB = 2096;
    long cpuPercent = 12;
    long bInMbps = 100;
    long bOutMbps = 100;
    long threads = 3;
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
    ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    zkCacheField.set(pulsarServices[0], mockCache);
    TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
    for (int i = 1; i <= 3; i++) {
        PulsarResourceDescription rd = createResourceDescription(memoryMB * i, cpuPercent * i, bInMbps * i, bOutMbps * 2, threads * i);
        ResourceUnit ru1 = new SimpleResourceUnit(String.format("http://prod1-broker%d.messaging.use.example.com:8080", i), rd);
        LoadRanker ranker = new ResourceAvailabilityRanker();
        long rank = ranker.getRank(rd);
        if (sortedRankingsInstance.containsKey(rank)) {
            // get the object set and put the rd in it
            sortedRankingsInstance.get(rank).add(ru1);
        } else {
            Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
            rus.add(ru1);
            sortedRankingsInstance.put(rank, rus);
        }
    }
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
    ar.set(sortedRankingsInstance);
    sortedRankings.set(loadManager, ar);
    int totalNamespaces = 10;
    Map<String, Integer> namespaceOwner = new HashMap<String, Integer>();
    for (int i = 0; i < totalNamespaces; i++) {
        ResourceUnit found = loadManager.getLeastLoaded(DestinationName.get("persistent://pulsar/use/primary-ns/topic" + i));
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 0);
        }
    }
    String inactiveBroker = "prod1-broker3.messaging.use.example.com:8080";
    // check owner list contains only two entries, broker-3 should not be in
    assertTrue(namespaceOwner.size() == 2);
    assertTrue(!namespaceOwner.containsKey(inactiveBroker));
}
Also used : PulsarResourceDescription(com.yahoo.pulsar.broker.loadbalance.impl.PulsarResourceDescription) Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) ResourceAvailabilityRanker(com.yahoo.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker) AtomicReference(java.util.concurrent.atomic.AtomicReference) TreeMap(java.util.TreeMap) SimpleResourceUnit(com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit) Field(java.lang.reflect.Field) SimpleResourceUnit(com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit) SimpleLoadManagerImpl(com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl) LocalZooKeeperCache(com.yahoo.pulsar.zookeeper.LocalZooKeeperCache) ZooKeeperCache(com.yahoo.pulsar.zookeeper.ZooKeeperCache) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Aggregations

SimpleLoadManagerImpl (com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl)6 PulsarResourceDescription (com.yahoo.pulsar.broker.loadbalance.impl.PulsarResourceDescription)4 ResourceAvailabilityRanker (com.yahoo.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker)4 SimpleResourceUnit (com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit)4 LocalZooKeeperCache (com.yahoo.pulsar.zookeeper.LocalZooKeeperCache)4 Field (java.lang.reflect.Field)4 HashSet (java.util.HashSet)4 TreeMap (java.util.TreeMap)4 AtomicReference (java.util.concurrent.atomic.AtomicReference)4 Test (org.testng.annotations.Test)4 ZooKeeperCache (com.yahoo.pulsar.zookeeper.ZooKeeperCache)3 HashMap (java.util.HashMap)3 Set (java.util.Set)3 NamespaceName (com.yahoo.pulsar.common.naming.NamespaceName)2 Map (java.util.Map)2 LeaderElectionService (com.yahoo.pulsar.broker.loadbalance.LeaderElectionService)1 LeaderListener (com.yahoo.pulsar.broker.loadbalance.LeaderElectionService.LeaderListener)1 LoadResourceQuotaUpdaterTask (com.yahoo.pulsar.broker.loadbalance.LoadResourceQuotaUpdaterTask)1 LoadSheddingTask (com.yahoo.pulsar.broker.loadbalance.LoadSheddingTask)1 BrokerService (com.yahoo.pulsar.broker.service.BrokerService)1