use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.
the class PulsarStats method updateStats.
public synchronized void updateStats(ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, PersistentTopic>>> topicsMap) {
StatsOutputStream topicStatsStream = new StatsOutputStream(tempTopicStatsBuf);
try {
tempMetricsCollection.clear();
bundleStats.clear();
brokerOperabilityMetrics.reset();
// Json begin
topicStatsStream.startObject();
topicsMap.forEach((namespaceName, bundles) -> {
if (bundles.isEmpty()) {
return;
}
try {
topicStatsStream.startObject(namespaceName);
nsStats.reset();
bundles.forEach((bundle, topics) -> {
NamespaceBundleStats currentBundleStats = bundleStats.computeIfAbsent(bundle, k -> new NamespaceBundleStats());
currentBundleStats.reset();
currentBundleStats.topics = topics.size();
topicStatsStream.startObject(NamespaceBundle.getBundleRange(bundle));
topicStatsStream.startObject("persistent");
topics.forEach((name, topic) -> {
try {
topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
} catch (Exception e) {
log.error("Failed to generate topic stats for topic {}: {}", name, e.getMessage(), e);
}
// this task: helps to activate inactive-backlog-cursors which have caught up and
// connected, also deactivate active-backlog-cursors which has backlog
topic.getManagedLedger().checkBackloggedCursors();
});
topicStatsStream.endObject();
topicStatsStream.endObject();
});
topicStatsStream.endObject();
// Update metricsCollection with namespace stats
tempMetricsCollection.add(nsStats.add(namespaceName));
} catch (Exception e) {
log.error("Failed to generate namespace stats for namespace {}: {}", namespaceName, e.getMessage(), e);
}
});
if (clusterReplicationMetrics.isMetricsEnabled()) {
clusterReplicationMetrics.get().forEach(clusterMetric -> tempMetricsCollection.add(clusterMetric));
clusterReplicationMetrics.reset();
}
brokerOperabilityMetrics.getMetrics().forEach(brokerOperabilityMetric -> tempMetricsCollection.add(brokerOperabilityMetric));
// json end
topicStatsStream.endObject();
} catch (Exception e) {
log.error("Unable to update destination stats", e);
}
// swap metricsCollection and tempMetricsCollection
List<Metrics> tempRefMetrics = metricsCollection;
metricsCollection = tempMetricsCollection;
tempMetricsCollection = tempRefMetrics;
bufferLock.writeLock().lock();
try {
ByteBuf tmp = topicStatsBuf;
topicStatsBuf = tempTopicStatsBuf;
tempTopicStatsBuf = tmp;
tempTopicStatsBuf.clear();
} finally {
bufferLock.writeLock().unlock();
}
}
use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.
the class LoadBalancerTest method testDestinationAssignmentWithExistingBundles.
/*
* Pre-publish load report to ZK, each broker has: - Difference memory capacity, for the first 3 brokers memory is
* bottleneck, for the 4/5th brokers CPU become bottleneck since memory is big enough - already has some bundles
* assigned Check the distribution of new destinations is roughly consistent (with <10% variation) with the ranking
*/
@Test
public void testDestinationAssignmentWithExistingBundles() throws Exception {
for (int i = 0; i < BROKER_COUNT; i++) {
ResourceQuota defaultQuota = new ResourceQuota();
defaultQuota.setMsgRateIn(20);
defaultQuota.setMsgRateOut(60);
defaultQuota.setBandwidthIn(20000);
defaultQuota.setBandwidthOut(60000);
defaultQuota.setMemory(87);
pulsarServices[i].getLocalZkCacheService().getResourceQuotaCache().setDefaultQuota(defaultQuota);
LoadReport lr = new LoadReport();
lr.setName(lookupAddresses[i]);
SystemResourceUsage sru = new SystemResourceUsage();
sru.setBandwidthIn(new ResourceUsage(0, 1024000));
sru.setBandwidthOut(new ResourceUsage(0, 1024000));
sru.setMemory(new ResourceUsage(0, 2048 * (i + 1)));
sru.setCpu(new ResourceUsage(60, 400));
lr.setSystemResourceUsage(sru);
Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
for (int j = 0; j < (i + 1) * 5; j++) {
String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
NamespaceBundleStats stats = new NamespaceBundleStats();
bundleStats.put(bundleName, stats);
}
lr.setBundleStats(bundleStats);
String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
String loadReportJson = objectMapper.writeValueAsString(lr);
bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
}
// sleep to wait load ranking be triggered
Thread.sleep(5000);
// print ranking
for (int i = 0; i < BROKER_COUNT; i++) {
AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRanking = getSortedRanking(pulsarServices[i]);
printSortedRanking(sortedRanking);
}
// check owner of new destiations and verify that the distribution is roughly
// consistent (variation < 10%) with the broker capacity:
int totalNamespaces = 250;
int[] expectedAssignments = new int[] { 17, 34, 51, 68, 85 };
Map<String, Integer> namespaceOwner = new HashMap<>();
for (int i = 0; i < totalNamespaces; i++) {
DestinationName fqdn = DestinationName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
ResourceUnit found = pulsarServices[0].getLoadManager().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(fqdn));
if (namespaceOwner.containsKey(found.getResourceId())) {
namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
} else {
namespaceOwner.put(found.getResourceId(), 1);
}
}
double expectedMaxVariation = 10.0;
for (int i = 0; i < BROKER_COUNT; i++) {
long actualValue = 0;
String resourceId = "http://" + lookupAddresses[i];
if (namespaceOwner.containsKey(resourceId)) {
actualValue = namespaceOwner.get(resourceId);
}
long expectedValue = expectedAssignments[i];
double variation = Math.abs(actualValue - expectedValue) * 100.0 / expectedValue;
log.info("Destination assignment - {}, actual: {}, expected baseline: {}, variation: {}/%", lookupAddresses[i], actualValue, expectedValue, String.format("%.2f", variation));
assertTrue(variation < expectedMaxVariation);
}
}
use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.
the class PersistentTopicE2ETest method testDeleteTopics.
@Test
public void testDeleteTopics() throws Exception {
BrokerService brokerService = pulsar.getBrokerService();
// 1. producers connect
Producer producer1 = pulsarClient.createProducer("persistent://prop/use/ns-abc/topic-1");
Producer producer2 = pulsarClient.createProducer("persistent://prop/use/ns-abc/topic-2");
brokerService.updateRates();
Map<String, NamespaceBundleStats> bundleStatsMap = brokerService.getBundleStats();
assertEquals(bundleStatsMap.size(), 1);
NamespaceBundleStats bundleStats = bundleStatsMap.get("prop/use/ns-abc/0x00000000_0xffffffff");
assertNotNull(bundleStats);
producer1.close();
admin.persistentTopics().delete("persistent://prop/use/ns-abc/topic-1");
brokerService.updateRates();
bundleStatsMap = brokerService.getBundleStats();
assertEquals(bundleStatsMap.size(), 1);
bundleStats = bundleStatsMap.get("prop/use/ns-abc/0x00000000_0xffffffff");
assertNotNull(bundleStats);
// // Delete 2nd topic as well
// producer2.close();
// admin.persistentTopics().delete("persistent://prop/use/ns-abc/topic-2");
//
// brokerService.updateRates();
//
// bundleStatsMap = brokerService.getBundleStats();
// assertEquals(bundleStatsMap.size(), 0);
}
use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.
the class LoadBalancerTest method writeLoadReportsForDynamicQuota.
private void writeLoadReportsForDynamicQuota(long timestamp) throws Exception {
for (int i = 0; i < BROKER_COUNT; i++) {
LoadReport lr = new LoadReport();
lr.setName(lookupAddresses[i]);
lr.setTimestamp(timestamp);
SystemResourceUsage sru = new SystemResourceUsage();
sru.setBandwidthIn(new ResourceUsage(5000 * (10 + i * 5), 1024000));
sru.setBandwidthOut(new ResourceUsage(15000 * (10 + i * 5), 1024000));
sru.setMemory(new ResourceUsage(25 * (10 + i * 5), 2048 * (i + 1)));
sru.setCpu(new ResourceUsage(200, 400));
lr.setSystemResourceUsage(sru);
Map<String, NamespaceBundleStats> bundleStats = new HashMap<String, NamespaceBundleStats>();
for (int j = 0; j < 5; j++) {
String bundleName = String.format("pulsar/use/primary-ns-%d-%d/0x00000000_0xffffffff", i, j);
NamespaceBundleStats stats = new NamespaceBundleStats();
stats.msgRateIn = 5 * (i + j);
stats.msgRateOut = 15 * (i + j);
stats.msgThroughputIn = 5000 * (i + j);
stats.msgThroughputOut = 15000 * (i + j);
stats.topics = 25 * (i + j);
stats.consumerCount = 50 * (i + j);
stats.producerCount = 50 * (i + j);
bundleStats.put(bundleName, stats);
}
lr.setBundleStats(bundleStats);
String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
String loadReportJson = objectMapper.writeValueAsString(lr);
bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
}
}
use of com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats in project pulsar by yahoo.
the class SimpleLoadManagerImpl method updateRealtimeResourceQuota.
private synchronized void updateRealtimeResourceQuota() {
long memObjectGroupSize = 500;
if (!currentLoadReports.isEmpty()) {
long totalBundles = 0;
long totalMemGroups = 0;
double totalMsgRateIn = 0.0;
double totalMsgRateOut = 0.0;
double totalMsgRate = 0.0;
double totalCpuUsage = 0.0;
double totalMemoryUsage = 0.0;
double totalBandwidthIn = 0.0;
double totalBandwidthOut = 0.0;
long loadReportTimestamp = -1;
// update resource factors
for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
LoadReport loadReport = entry.getValue();
if (loadReport.getTimestamp() > loadReportTimestamp) {
loadReportTimestamp = loadReport.getTimestamp();
}
Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
if (bundleStats == null) {
continue;
}
for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
totalBundles++;
NamespaceBundleStats stats = statsEntry.getValue();
totalMemGroups += (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
totalBandwidthIn += stats.msgThroughputIn;
totalBandwidthOut += stats.msgThroughputOut;
}
SystemResourceUsage resUsage = loadReport.getSystemResourceUsage();
totalMsgRateIn += loadReport.getMsgRateIn();
totalMsgRateOut += loadReport.getMsgRateOut();
totalCpuUsage = totalCpuUsage + resUsage.getCpu().usage;
totalMemoryUsage = totalMemoryUsage + resUsage.getMemory().usage;
}
totalMsgRate = totalMsgRateIn + totalMsgRateOut;
long timePast = loadReportTimestamp - this.lastResourceQuotaUpdateTimestamp;
this.lastResourceQuotaUpdateTimestamp = loadReportTimestamp;
if (totalMsgRate > 1000 && totalMemGroups > 30) {
this.realtimeCpuLoadFactor = timeSmoothValue(this.realtimeCpuLoadFactor, totalCpuUsage / totalMsgRate, RESOURCE_QUOTA_MIN_CPU_FACTOR, RESOURCE_QUOTA_MAX_CPU_FACTOR, timePast);
this.realtimeMemoryLoadFactor = timeSmoothValue(this.realtimeMemoryLoadFactor, totalMemoryUsage / totalMemGroups, RESOURCE_QUOTA_MIN_MEM_FACTOR, RESOURCE_QUOTA_MAX_MEM_FACTOR, timePast);
}
// calculate average bundle
if (totalBundles > 30 && this.realtimeAvgResourceQuota.getDynamic()) {
ResourceQuota oldQuota = this.realtimeAvgResourceQuota;
ResourceQuota newQuota = timeSmoothQuota(oldQuota, totalMsgRateIn / totalBundles, totalMsgRateOut / totalBundles, totalBandwidthIn / totalBundles, totalBandwidthOut / totalBundles, totalMemoryUsage / totalBundles, timePast);
this.realtimeAvgResourceQuota = newQuota;
}
// update realtime quota for each bundle
Map<String, ResourceQuota> newQuotas = new HashMap<>();
for (Map.Entry<ResourceUnit, LoadReport> entry : currentLoadReports.entrySet()) {
ResourceUnit resourceUnit = entry.getKey();
LoadReport loadReport = entry.getValue();
Map<String, NamespaceBundleStats> bundleStats = loadReport.getBundleStats();
if (bundleStats == null) {
continue;
}
for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
String bundle = statsEntry.getKey();
NamespaceBundleStats stats = statsEntry.getValue();
long memGroupCount = (1 + (stats.topics + stats.producerCount + stats.consumerCount) / memObjectGroupSize);
double newMemoryQuota = memGroupCount * this.realtimeMemoryLoadFactor;
ResourceQuota oldQuota = getResourceQuota(bundle);
ResourceQuota newQuota = timeSmoothQuota(oldQuota, stats.msgRateIn, stats.msgRateOut, stats.msgThroughputIn, stats.msgThroughputOut, newMemoryQuota, timePast);
newQuotas.put(bundle, newQuota);
}
}
this.realtimeResourceQuotas.set(newQuotas);
}
}
Aggregations