use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class LoadBalancerTest method testLoadBalanceDiscardingInactiveBrokersInSelection.
/**
* This test puts few brokers in the ranking that load balance uses but does not put few brokers in the mock
* zookeeper cache, if the broker is not in the zk cache, it's considered inactive.
*
* We should not see any of these inactive brokers assigned any namespace.
*/
@Test(enabled = false)
void testLoadBalanceDiscardingInactiveBrokersInSelection() throws Exception {
long memoryMB = 2096;
long cpuPercent = 12;
long bInMbps = 100;
long bOutMbps = 100;
long threads = 3;
LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080");
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
zkCacheField.setAccessible(true);
zkCacheField.set(pulsarServices[0], mockCache);
TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
for (int i = 1; i <= 3; i++) {
PulsarResourceDescription rd = createResourceDescription(memoryMB * i, cpuPercent * i, bInMbps * i, bOutMbps * 2, threads * i);
ResourceUnit ru1 = new SimpleResourceUnit(String.format("http://prod1-broker%d.messaging.use.example.com:8080", i), rd);
LoadRanker ranker = new ResourceAvailabilityRanker();
long rank = ranker.getRank(rd);
if (sortedRankingsInstance.containsKey(rank)) {
// get the object set and put the rd in it
sortedRankingsInstance.get(rank).add(ru1);
} else {
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
sortedRankingsInstance.put(rank, rus);
}
}
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
ar.set(sortedRankingsInstance);
sortedRankings.set(loadManager, ar);
int totalNamespaces = 10;
Map<String, Integer> namespaceOwner = new HashMap<String, Integer>();
for (int i = 0; i < totalNamespaces; i++) {
ResourceUnit found = loadManager.getLeastLoaded(TopicName.get("persistent://pulsar/use/primary-ns/topic" + i)).get();
if (namespaceOwner.containsKey(found.getResourceId())) {
namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
} else {
namespaceOwner.put(found.getResourceId(), 0);
}
}
String inactiveBroker = "prod1-broker3.messaging.use.example.com:8080";
// check owner list contains only two entries, broker-3 should not be in
assertTrue(namespaceOwner.size() == 2);
assertTrue(!namespaceOwner.containsKey(inactiveBroker));
}
use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class LoadBalancerTest method testGetLeastLoadedBasic.
@Test(enabled = false)
public void testGetLeastLoadedBasic() throws Exception {
LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
zkCacheField.setAccessible(true);
LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[0]);
LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[1]);
zkCacheField.set(pulsarServices[0], mockCache);
zkCacheField.set(pulsarServices[1], mockCache);
LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
// TODO move to its own test
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
ResourceUnit ru1 = new SimpleResourceUnit("http://prod1-broker1.messaging.use.example.com:8080", rd);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
LoadRanker lr = new ResourceAvailabilityRanker();
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
sortedRankingsInstance.get().put(lr.getRank(rd), rus);
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
sortedRankings.set(loadManager, sortedRankingsInstance);
ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
assertEquals("http://prod1-broker1.messaging.use.example.com:8080", found.getResourceId());
zkCacheField.set(pulsarServices[0], originalLZK1);
zkCacheField.set(pulsarServices[1], originalLZK2);
}
use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class LoadBalancerTest method createResourceDescription.
/*
* creates a ResourceDescription where the max limits for different parameters are as below
*
* Memory = 16GB cpu = 100 percent bandwidthIn = 1Gbps bandwidthOut = 1Gbps threads = 100
*/
private PulsarResourceDescription createResourceDescription(long memoryInMB, long cpuPercentage, long bandwidthInMbps, long bandwidthOutInMbps, long threads) {
long KB = 1024;
long MB = 1024 * KB;
long GB = 1024 * MB;
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(memoryInMB, 4 * GB));
rd.put("cpu", new ResourceUsage(cpuPercentage, 100));
rd.put("bandwidthIn", new ResourceUsage(bandwidthInMbps * MB, GB));
rd.put("bandwidthOut", new ResourceUsage(bandwidthOutInMbps * MB, GB));
return rd;
}
use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class SimpleLoadManagerImplTest method testPrimarySecondary.
@Test(enabled = false)
public void testPrimarySecondary() throws Exception {
createNamespacePolicies(pulsar1);
LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
ZooKeeperChildrenCache zooKeeperChildrenCache = mock(ZooKeeperChildrenCache.class);
Set<String> activeBrokers = Sets.newHashSet("prod2-broker7.messaging.use.example.com:8080", "prod2-broker8.messaging.use.example.com:8080", "prod2-broker9.messaging.use.example.com:8080");
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
when(zooKeeperChildrenCache.get()).thenReturn(activeBrokers);
when(zooKeeperChildrenCache.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
zkCacheField.setAccessible(true);
LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsar1);
LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsar2);
log.info("lzk are {} 2: {}", originalLZK1.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT), originalLZK2.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
zkCacheField.set(pulsar1, mockCache);
LocalZooKeeperCache newZk = (LocalZooKeeperCache) pulsar1.getLocalZkCache();
log.info("lzk mocked are {}", newZk.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
ZooKeeperChildrenCache availableActiveBrokers = new ZooKeeperChildrenCache(pulsar1.getLocalZkCache(), SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT);
log.info("lzk mocked active brokers are {}", availableActiveBrokers.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
LoadRanker lr = new ResourceAvailabilityRanker();
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
sortedRankingsInstance.get().put(lr.getRank(rd), rus);
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
sortedRankings.set(loadManager, sortedRankingsInstance);
ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
assertEquals(found.getResourceId(), ru1.getResourceId());
zkCacheField.set(pulsar1, originalLZK1);
}
use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class SimpleLoadManagerImplTest method testPrimary.
@Test(enabled = true)
public void testPrimary() throws Exception {
createNamespacePolicies(pulsar1);
LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
ResourceUnit ru1 = new SimpleResourceUnit("http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort(), rd);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
LoadRanker lr = new ResourceAvailabilityRanker();
// inject the load report and rankings
Map<ResourceUnit, org.apache.pulsar.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
loadReport.setSystemResourceUsage(new SystemResourceUsage());
loadReports.put(ru1, loadReport);
setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), new HashSet<String>(), new ResourceQuota(), new HashSet<String>(), new ResourceQuota());
Map<ResourceUnit, ResourceUnitRanking> rankings = new HashMap<>();
rankings.put(ru1, ranking);
setObjectField(SimpleLoadManagerImpl.class, loadManager, "resourceUnitRankings", rankings);
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
sortedRankingsInstance.get().put(lr.getRank(rd), rus);
setObjectField(SimpleLoadManagerImpl.class, loadManager, "sortedRankings", sortedRankingsInstance);
ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
// broker is not active so found should be null
assertNotEquals(found, null, "did not find a broker when expected one to be found");
}
Aggregations