Examples with Metrics com.yahoo.pulsar.broker.stats.Metrics used on opensource projects

Example 1 with Metrics

use of com.yahoo.pulsar.broker.stats.Metrics in project pulsar by yahoo.

the class PulsarStats method updateStats.

public synchronized void updateStats(ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, PersistentTopic>>> topicsMap) {
    StatsOutputStream topicStatsStream = new StatsOutputStream(tempTopicStatsBuf);
    try {
        tempMetricsCollection.clear();
        bundleStats.clear();
        brokerOperabilityMetrics.reset();
        // Json begin
        topicStatsStream.startObject();
        topicsMap.forEach((namespaceName, bundles) -> {
            if (bundles.isEmpty()) {
                return;
            }
            try {
                topicStatsStream.startObject(namespaceName);
                nsStats.reset();
                bundles.forEach((bundle, topics) -> {
                    NamespaceBundleStats currentBundleStats = bundleStats.computeIfAbsent(bundle, k -> new NamespaceBundleStats());
                    currentBundleStats.reset();
                    currentBundleStats.topics = topics.size();
                    topicStatsStream.startObject(NamespaceBundle.getBundleRange(bundle));
                    topicStatsStream.startObject("persistent");
                    topics.forEach((name, topic) -> {
                        try {
                            topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                        } catch (Exception e) {
                            log.error("Failed to generate topic stats for topic {}: {}", name, e.getMessage(), e);
                        }
                        // this task: helps to activate inactive-backlog-cursors which have caught up and
                        // connected, also deactivate active-backlog-cursors which has backlog
                        topic.getManagedLedger().checkBackloggedCursors();
                    });
                    topicStatsStream.endObject();
                    topicStatsStream.endObject();
                });
                topicStatsStream.endObject();
                // Update metricsCollection with namespace stats
                tempMetricsCollection.add(nsStats.add(namespaceName));
            } catch (Exception e) {
                log.error("Failed to generate namespace stats for namespace {}: {}", namespaceName, e.getMessage(), e);
            }
        });
        if (clusterReplicationMetrics.isMetricsEnabled()) {
            clusterReplicationMetrics.get().forEach(clusterMetric -> tempMetricsCollection.add(clusterMetric));
            clusterReplicationMetrics.reset();
        }
        brokerOperabilityMetrics.getMetrics().forEach(brokerOperabilityMetric -> tempMetricsCollection.add(brokerOperabilityMetric));
        // json end
        topicStatsStream.endObject();
    } catch (Exception e) {
        log.error("Unable to update destination stats", e);
    }
    // swap metricsCollection and tempMetricsCollection
    List<Metrics> tempRefMetrics = metricsCollection;
    metricsCollection = tempMetricsCollection;
    tempMetricsCollection = tempRefMetrics;
    bufferLock.writeLock().lock();
    try {
        ByteBuf tmp = topicStatsBuf;
        topicStatsBuf = tempTopicStatsBuf;
        tempTopicStatsBuf = tmp;
        tempTopicStatsBuf.clear();
    } finally {
        bufferLock.writeLock().unlock();
    }
}

Example 2 with Metrics

use of com.yahoo.pulsar.broker.stats.Metrics in project pulsar by yahoo.

the class ManagedLedgerMetrics method groupLedgersByDimension.

/**
     * Build a map of dimensions key to list of destination stats (not thread-safe)
     * <p>
     * 
     * @return
     */
private Map<Metrics, List<ManagedLedgerImpl>> groupLedgersByDimension() {
    ledgersByDimensionMap.clear();
    for (Entry<String, ManagedLedgerImpl> e : getManagedLedgers().entrySet()) {
        String ledgerName = e.getKey();
        ManagedLedgerImpl ledger = e.getValue();
        // we want to aggregate by NS dimension
        String namespace = parseNamespaceFromLedgerName(ledgerName);
        Metrics metrics = createMetricsByDimension(namespace);
        populateDimensionMap(ledgersByDimensionMap, metrics, ledger);
    }
    return ledgersByDimensionMap;
}

Example 3 with Metrics

use of com.yahoo.pulsar.broker.stats.Metrics in project pulsar by yahoo.

the class ManagedLedgerMetrics method aggregate.

/**
     * Aggregation by namespace (not thread-safe)
     * 
     * @param ledgersByDimension
     * @return
     */
private List<Metrics> aggregate(Map<Metrics, List<ManagedLedgerImpl>> ledgersByDimension) {
    metricsCollection.clear();
    for (Entry<Metrics, List<ManagedLedgerImpl>> e : ledgersByDimension.entrySet()) {
        Metrics metrics = e.getKey();
        List<ManagedLedgerImpl> ledgers = e.getValue();
        // prepare aggregation map
        tempAggregatedMetricsMap.clear();
        for (ManagedLedgerImpl ledger : ledgers) {
            ManagedLedgerMXBean lStats = ledger.getStats();
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryBytesRate", lStats.getAddEntryBytesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryErrors", (double) lStats.getAddEntryErrors());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryMessagesRate", lStats.getAddEntryMessagesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntrySucceed", (double) lStats.getAddEntrySucceed());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_NumberOfMessagesInBacklog", (double) lStats.getNumberOfMessagesInBacklog());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesBytesRate", lStats.getReadEntriesBytesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesErrors", (double) lStats.getReadEntriesErrors());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesRate", lStats.getReadEntriesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesSucceeded", (double) lStats.getReadEntriesSucceeded());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_StoredMessagesSize", (double) lStats.getStoredMessagesSize());
            // handle bucket entries initialization here
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_AddEntryLatencyBuckets", ENTRY_LATENCY_BUCKETS_MS, lStats.getAddEntryLatencyBuckets());
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_LedgerSwitchLatencyBuckets", ENTRY_LATENCY_BUCKETS_MS, lStats.getLedgerSwitchLatencyBuckets());
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_EntrySizeBuckets", ENTRY_SIZE_BUCKETS_BYTES, lStats.getEntrySizeBuckets());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_MarkDeleteRate", lStats.getMarkDeleteRate());
        }
        for (Entry<String, Double> ma : tempAggregatedMetricsMap.entrySet()) {
            metrics.put(ma.getKey(), ma.getValue());
        }
        metricsCollection.add(metrics);
    }
    return metricsCollection;
}

Example 4 with Metrics

use of com.yahoo.pulsar.broker.stats.Metrics in project pulsar by yahoo.

the class PersistentTopicE2ETest method testBrokerTopicStats.

@Test
public void testBrokerTopicStats() throws Exception {
    BrokerService brokerService = this.pulsar.getBrokerService();
    Field field = BrokerService.class.getDeclaredField("statsUpdater");
    field.setAccessible(true);
    ScheduledExecutorService statsUpdater = (ScheduledExecutorService) field.get(brokerService);
    // disable statsUpdate to calculate rates explicitly
    statsUpdater.shutdown();
    final String namespace = "prop/use/ns-abc";
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://" + namespace + "/topic0", producerConf);
    // 1. producer publish messages
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Metrics metric = null;
    // sleep 1 sec to caclulate metrics per second
    Thread.sleep(1000);
    brokerService.updateRates();
    List<Metrics> metrics = brokerService.getDestinationMetrics();
    for (int i = 0; i < metrics.size(); i++) {
        if (metrics.get(i).getDimension("namespace").equalsIgnoreCase(namespace)) {
            metric = metrics.get(i);
            break;
        }
    }
    assertNotNull(metric);
    double msgInRate = (double) metrics.get(0).getMetrics().get("brk_in_rate");
    // rate should be calculated and no must be > 0 as we have produced 10 msgs so far
    assertTrue(msgInRate > 0);
}

Example 5 with Metrics

use of com.yahoo.pulsar.broker.stats.Metrics in project pulsar by yahoo.

the class SimpleLoadManagerImpl method updateLoadBalancingMetrics.

private void updateLoadBalancingMetrics(String hostname, long finalRank, ResourceUnitRanking ranking) {
    List<Metrics> metrics = Lists.newArrayList();
    Map<String, String> dimensions = new HashMap<>();
    dimensions.put("broker", hostname);
    Metrics m = Metrics.create(dimensions);
    m.put("brk_lb_load_rank", finalRank);
    m.put("brk_lb_quota_pct_cpu", ranking.getAllocatedLoadPercentageCPU());
    m.put("brk_lb_quota_pct_memory", ranking.getAllocatedLoadPercentageMemory());
    m.put("brk_lb_quota_pct_bandwidth_in", ranking.getAllocatedLoadPercentageBandwidthIn());
    m.put("brk_lb_quota_pct_bandwidth_out", ranking.getAllocatedLoadPercentageBandwidthOut());
    metrics.add(m);
    this.loadBalancingMetrics.set(metrics);
}