use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class LoadBalancerTest method testLoadBalanceDistributionAmongUnequallyLoaded.
@Test(enabled = false)
void testLoadBalanceDistributionAmongUnequallyLoaded() throws Exception {
long memoryMB = 4096;
long cpuPercent = 25;
long bInMbps = 256;
long bOutMbps = 256;
long threads = 25;
LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
zkCacheField.setAccessible(true);
zkCacheField.set(pulsarServices[0], mockCache);
int totalAvailabilityWeight = 0;
TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
for (int i = 1; i <= 3; i++) {
PulsarResourceDescription rd = createResourceDescription(memoryMB * i, cpuPercent * i, bInMbps * i, bOutMbps * 2, threads * i);
ResourceUnit ru1 = new SimpleResourceUnit(String.format("http://prod1-broker%d.messaging.use.example.com:8080", i), rd);
LoadRanker ranker = new ResourceAvailabilityRanker();
long rank = ranker.getRank(rd);
if (sortedRankingsInstance.containsKey(rank)) {
// get the object set and put the rd in it
sortedRankingsInstance.get(rank).add(ru1);
} else {
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
sortedRankingsInstance.put(rank, rus);
}
totalAvailabilityWeight += rank;
}
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
ar.set(sortedRankingsInstance);
sortedRankings.set(loadManager, ar);
int totalNamespaces = 1000;
Map<String, Integer> namespaceOwner = new HashMap<String, Integer>();
for (int i = 0; i < totalNamespaces; i++) {
ResourceUnit found = loadManager.getLeastLoaded(TopicName.get("persistent://pulsar/use/primary-ns/topic-" + i)).get();
if (namespaceOwner.containsKey(found.getResourceId())) {
namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
} else {
namespaceOwner.put(found.getResourceId(), 0);
}
}
for (Map.Entry<Long, Set<ResourceUnit>> entry : sortedRankingsInstance.entrySet()) {
int selectionProbability = (int) (((double) entry.getKey() / totalAvailabilityWeight) * 100);
int idealExpectedOwned = selectionProbability * (int) ((double) totalNamespaces / 100);
int expectedOwnedLowerBound = idealExpectedOwned - idealExpectedOwned / 10;
int expectedOwnedUpperBound = idealExpectedOwned + idealExpectedOwned / 10;
for (ResourceUnit ru : entry.getValue()) {
assertTrue(namespaceOwner.containsKey(ru.getResourceId()));
int ownedNamespaces = namespaceOwner.get(ru.getResourceId());
assertTrue(ownedNamespaces > expectedOwnedLowerBound || ownedNamespaces < expectedOwnedUpperBound);
}
}
}
use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class LoadBalancerTest method testLoadbalanceDistributionAmongEquallyLoaded.
/*
* Simple Test, creates three Resource Units (brokers) with equal load and expects that out of 1005 namespaces it
* should be divided fairly equally with about 10% of variation
*
*/
@Test(enabled = false)
public void testLoadbalanceDistributionAmongEquallyLoaded() throws Exception {
LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
ZooKeeperCache mockCache = mock(ZooKeeperCache.class);
Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
zkCacheField.setAccessible(true);
zkCacheField.set(pulsarServices[0], mockCache);
long memoryMB = 4096;
long cpuPercent = 45;
long bInMbps = 350;
long bOutMbps = 180;
long threads = 10;
// TODO move to its own test
PulsarResourceDescription rd = createResourceDescription(memoryMB, cpuPercent, bInMbps, bOutMbps, threads);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
for (String broker : activeBrokers) {
ResourceUnit ru = new SimpleResourceUnit(broker, rd);
rus.add(ru);
}
TreeMap<Long, Set<ResourceUnit>> sortedRankingsInstance = new TreeMap<Long, Set<ResourceUnit>>();
LoadRanker ranker = new ResourceAvailabilityRanker();
sortedRankingsInstance.put(ranker.getRank(rd), rus);
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
AtomicReference<TreeMap<Long, Set<ResourceUnit>>> ar = new AtomicReference<TreeMap<Long, Set<ResourceUnit>>>();
ar.set(sortedRankingsInstance);
sortedRankings.set(loadManager, ar);
}
use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class SimpleLoadManagerImplTest method testDoLoadShedding.
@Test(enabled = true)
public void testDoLoadShedding() throws Exception {
LoadManager loadManager = spy(new SimpleLoadManagerImpl(pulsar1));
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
ResourceUnit ru1 = new SimpleResourceUnit("http://pulsar-broker1.com:8080", rd);
ResourceUnit ru2 = new SimpleResourceUnit("http://pulsar-broker2.com:8080", rd);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
rus.add(ru2);
LoadRanker lr = new ResourceAvailabilityRanker();
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
sortedRankingsInstance.get().put(lr.getRank(rd), rus);
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
sortedRankings.set(loadManager, sortedRankingsInstance);
// inject the load report and rankings
SystemResourceUsage systemResource = new SystemResourceUsage();
systemResource.setBandwidthIn(new ResourceUsage(90, 100));
Map<String, NamespaceBundleStats> stats = Maps.newHashMap();
NamespaceBundleStats nsb1 = new NamespaceBundleStats();
nsb1.msgRateOut = 10000;
NamespaceBundleStats nsb2 = new NamespaceBundleStats();
nsb2.msgRateOut = 10000;
stats.put("property/cluster/namespace1/0x00000000_0xFFFFFFFF", nsb1);
stats.put("property/cluster/namespace2/0x00000000_0xFFFFFFFF", nsb2);
Map<ResourceUnit, org.apache.pulsar.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport1 = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
loadReport1.setSystemResourceUsage(systemResource);
loadReport1.setBundleStats(stats);
org.apache.pulsar.policies.data.loadbalancer.LoadReport loadReport2 = new org.apache.pulsar.policies.data.loadbalancer.LoadReport();
loadReport2.setSystemResourceUsage(new SystemResourceUsage());
loadReport2.setBundleStats(stats);
loadReports.put(ru1, loadReport1);
loadReports.put(ru2, loadReport2);
setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
((SimpleLoadManagerImpl) loadManager).doLoadShedding();
verify(loadManager, atLeastOnce()).doLoadShedding();
}
use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class SimpleLoadManagerImplTest method testBasicBrokerSelection.
@Test(enabled = true)
public void testBasicBrokerSelection() throws Exception {
LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
LoadRanker lr = new ResourceAvailabilityRanker();
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
sortedRankingsInstance.get().put(lr.getRank(rd), rus);
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
sortedRankings.set(loadManager, sortedRankingsInstance);
Optional<ResourceUnit> res = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10"));
// broker is not active so found should be null
assertEquals(res, Optional.empty(), "found a broker when expected none to be found");
}
use of org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription in project incubator-pulsar by apache.
the class SimpleLoadManagerImplTest method testResourceDescription.
@Test
public void testResourceDescription() {
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
PulsarResourceDescription rd1 = new PulsarResourceDescription();
rd1.put("memory", new ResourceUsage(2048, 4096));
rd1.put("cpu", new ResourceUsage(50, 100));
rd1.put("bandwidthIn", new ResourceUsage(550 * 1024, 1024 * 1024));
rd1.put("bandwidthOut", new ResourceUsage(850 * 1024, 1024 * 1024));
assertTrue(rd.compareTo(rd1) == 1);
assertTrue(rd1.calculateRank() > rd.calculateRank());
SimpleLoadCalculatorImpl calc = new SimpleLoadCalculatorImpl();
calc.recaliberateResourceUsagePerServiceUnit(null);
assertNull(calc.getResourceDescription(null));
}
Aggregations