use of com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage in project pulsar by yahoo.
the class SimpleLoadManagerImplTest method testBasicBrokerSelection.
@Test(enabled = true)
public void testBasicBrokerSelection() throws Exception {
LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
LoadRanker lr = new ResourceAvailabilityRanker();
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
sortedRankingsInstance.get().put(lr.getRank(rd), rus);
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
sortedRankings.set(loadManager, sortedRankingsInstance);
ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(new NamespaceName("pulsar/use/primary-ns.10"));
// broker is not active so found should be null
assertEquals(found, null, "found a broker when expected none to be found");
}
use of com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage in project pulsar by yahoo.
the class LinuxBrokerHostUsageImpl method calculateBrokerHostUsage.
private void calculateBrokerHostUsage() {
List<String> nics = getNics();
double totalNicLimit = getTotalNicLimitKbps(nics);
double totalNicUsageTx = getTotalNicUsageTxKb(nics);
double totalNicUsageRx = getTotalNicUsageRxKb(nics);
double totalCpuLimit = getTotalCpuLimit();
CpuStat cpuStat = getTotalCpuUsage();
SystemResourceUsage usage = new SystemResourceUsage();
long now = System.currentTimeMillis();
if (lastCollection == 0L) {
usage.setMemory(getMemUsage());
usage.setBandwidthIn(new ResourceUsage(0d, totalNicLimit));
usage.setBandwidthOut(new ResourceUsage(0d, totalNicLimit));
usage.setCpu(new ResourceUsage(0d, totalCpuLimit));
} else {
double elapsedSeconds = (now - lastCollection) / 1000d;
double nicUsageTx = (totalNicUsageTx - lastTotalNicUsageTx) / elapsedSeconds;
double nicUsageRx = (totalNicUsageRx - lastTotalNicUsageRx) / elapsedSeconds;
if (cpuStat != null && lastCpuStat != null) {
// we need two non null stats to get a usage report
long cpuTimeDiff = cpuStat.getTotalTime() - lastCpuStat.getTotalTime();
long cpuUsageDiff = cpuStat.getUsage() - lastCpuStat.getUsage();
double cpuUsage = ((double) cpuUsageDiff / (double) cpuTimeDiff) * totalCpuLimit;
usage.setCpu(new ResourceUsage(cpuUsage, totalCpuLimit));
}
usage.setMemory(getMemUsage());
usage.setBandwidthIn(new ResourceUsage(nicUsageRx, totalNicLimit));
usage.setBandwidthOut(new ResourceUsage(nicUsageTx, totalNicLimit));
}
lastTotalNicUsageTx = totalNicUsageTx;
lastTotalNicUsageRx = totalNicUsageRx;
lastCpuStat = cpuStat;
lastCollection = System.currentTimeMillis();
this.usage = usage;
}
use of com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage in project pulsar by yahoo.
the class LoadBalancerTest method testUpdateLoadReportAndCheckUpdatedRanking.
/*
* tests rankings get updated when we write write the new load reports to the zookeeper on loadbalance root node
* tests writing pre-configured load report on the zookeeper translates the pre-calculated rankings
*/
@Test
public void testUpdateLoadReportAndCheckUpdatedRanking() throws Exception {
for (int i = 0; i < BROKER_COUNT; i++) {
LoadReport lr = new LoadReport();
lr.setName(lookupAddresses[i]);
SystemResourceUsage sru = new SystemResourceUsage();
sru.setBandwidthIn(new ResourceUsage(256, 1024000));
sru.setBandwidthOut(new ResourceUsage(250, 1024000));
sru.setMemory(new ResourceUsage(1024, 8192));
sru.setCpu(new ResourceUsage(5, 400));
lr.setSystemResourceUsage(sru);
String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
String loadReportJson = objectMapper.writeValueAsString(lr);
bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
}
// sleep to wait the load ranking be triggered
Thread.sleep(5000);
// do lookup for bunch of bundles
int totalNamespaces = 200;
Map<String, Integer> namespaceOwner = new HashMap<>();
for (int i = 0; i < totalNamespaces; i++) {
DestinationName fqdn = DestinationName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
ResourceUnit found = pulsarServices[0].getLoadManager().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(fqdn));
if (namespaceOwner.containsKey(found.getResourceId())) {
namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
} else {
namespaceOwner.put(found.getResourceId(), 1);
}
}
// assert that distribution variation is not more than 10%
int averageNamespaces = totalNamespaces / BROKER_COUNT;
int tenPercentOfAverageNamespaces = averageNamespaces / 10;
int lowerBound = averageNamespaces - tenPercentOfAverageNamespaces;
int upperBound = averageNamespaces + tenPercentOfAverageNamespaces;
// assert each broker received ownership of fair amount of namespaces 90%+
for (Map.Entry<String, Integer> broker : namespaceOwner.entrySet()) {
log.info("Count of bundles assigned: {}, {}", broker.getKey(), broker.getValue());
assertTrue(broker.getValue() >= lowerBound && broker.getValue() <= upperBound);
}
}
use of com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage in project pulsar by yahoo.
the class LoadBalancerTest method testGetLeastLoadedBasic.
@Test(enabled = false)
public void testGetLeastLoadedBasic() throws Exception {
LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
Set<String> activeBrokers = Sets.newHashSet("prod1-broker1.messaging.use.example.com:8080", "prod1-broker2.messaging.use.example.com:8080", "prod1-broker3.messaging.use.example.com:8080");
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
zkCacheField.setAccessible(true);
LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[0]);
LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsarServices[1]);
zkCacheField.set(pulsarServices[0], mockCache);
zkCacheField.set(pulsarServices[1], mockCache);
LoadManager loadManager = new SimpleLoadManagerImpl(pulsarServices[0]);
// TODO move to its own test
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
ResourceUnit ru1 = new SimpleResourceUnit("http://prod1-broker1.messaging.use.example.com:8080", rd);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
LoadRanker lr = new ResourceAvailabilityRanker();
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
sortedRankingsInstance.get().put(lr.getRank(rd), rus);
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
sortedRankings.set(loadManager, sortedRankingsInstance);
ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(new NamespaceName("pulsar/use/primary-ns.10"));
assertEquals("http://prod1-broker1.messaging.use.example.com:8080", found.getResourceId());
zkCacheField.set(pulsarServices[0], originalLZK1);
zkCacheField.set(pulsarServices[1], originalLZK2);
}
use of com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage in project pulsar by yahoo.
the class SimpleLoadManagerImplTest method testPrimarySecondary.
@Test(enabled = false)
public void testPrimarySecondary() throws Exception {
LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
ZooKeeperChildrenCache zooKeeperChildrenCache = mock(ZooKeeperChildrenCache.class);
Set<String> activeBrokers = Sets.newHashSet("prod2-broker7.messaging.use.example.com:8080", "prod2-broker8.messaging.use.example.com:8080", "prod2-broker9.messaging.use.example.com:8080");
when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
when(zooKeeperChildrenCache.get()).thenReturn(activeBrokers);
when(zooKeeperChildrenCache.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
zkCacheField.setAccessible(true);
LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsar1);
LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsar2);
log.info("lzk are {} 2: {}", originalLZK1.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT), originalLZK2.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
zkCacheField.set(pulsar1, mockCache);
LocalZooKeeperCache newZk = (LocalZooKeeperCache) pulsar1.getLocalZkCache();
log.info("lzk mocked are {}", newZk.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
ZooKeeperChildrenCache availableActiveBrokers = new ZooKeeperChildrenCache(pulsar1.getLocalZkCache(), SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT);
log.info("lzk mocked active brokers are {}", availableActiveBrokers.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
PulsarResourceDescription rd = new PulsarResourceDescription();
rd.put("memory", new ResourceUsage(1024, 4096));
rd.put("cpu", new ResourceUsage(10, 100));
rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
rus.add(ru1);
LoadRanker lr = new ResourceAvailabilityRanker();
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
sortedRankingsInstance.get().put(lr.getRank(rd), rus);
Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
sortedRankings.setAccessible(true);
sortedRankings.set(loadManager, sortedRankingsInstance);
ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(new NamespaceName("pulsar/use/primary-ns.10"));
assertEquals(found.getResourceId(), ru1.getResourceId());
zkCacheField.set(pulsar1, originalLZK1);
}
Aggregations