Examples with Metrics org.apache.pulsar.common.stats.Metrics used on opensource projects

Example 1 with Metrics

use of org.apache.pulsar.common.stats.Metrics in project incubator-pulsar by apache.

the class ProxyPublishConsumeTest method verifyProxyMetrics.

private void verifyProxyMetrics(Client client, String baseUrl) {
    // generate metrics
    service.getProxyStats().generate();
    // collect metrics
    String statUrl = baseUrl + "metrics";
    WebTarget webTarget = client.target(statUrl);
    Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_JSON);
    Response response = (Response) invocationBuilder.get();
    String responseStr = response.readEntity(String.class);
    final Gson gson = new Gson();
    final List<Metrics> data = gson.fromJson(responseStr, new TypeToken<List<Metrics>>() {
    }.getType());
    Assert.assertFalse(data.isEmpty());
}

Example 2 with Metrics

use of org.apache.pulsar.common.stats.Metrics in project incubator-pulsar by apache.

the class PulsarStats method updateStats.

public synchronized void updateStats(ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, Topic>>> topicsMap) {
    StatsOutputStream topicStatsStream = new StatsOutputStream(tempTopicStatsBuf);
    try {
        tempMetricsCollection.clear();
        bundleStats.clear();
        brokerOperabilityMetrics.reset();
        // Json begin
        topicStatsStream.startObject();
        topicsMap.forEach((namespaceName, bundles) -> {
            if (bundles.isEmpty()) {
                return;
            }
            try {
                topicStatsStream.startObject(namespaceName);
                nsStats.reset();
                bundles.forEach((bundle, topics) -> {
                    NamespaceBundleStats currentBundleStats = bundleStats.computeIfAbsent(bundle, k -> new NamespaceBundleStats());
                    currentBundleStats.reset();
                    currentBundleStats.topics = topics.size();
                    topicStatsStream.startObject(NamespaceBundle.getBundleRange(bundle));
                    tempNonPersistentTopics.clear();
                    // start persistent topic
                    topicStatsStream.startObject("persistent");
                    topics.forEach((name, topic) -> {
                        if (topic instanceof PersistentTopic) {
                            try {
                                topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                            } catch (Exception e) {
                                log.error("Failed to generate topic stats for topic {}: {}", name, e.getMessage(), e);
                            }
                            // this task: helps to activate inactive-backlog-cursors which have caught up and
                            // connected, also deactivate active-backlog-cursors which has backlog
                            ((PersistentTopic) topic).getManagedLedger().checkBackloggedCursors();
                        } else if (topic instanceof NonPersistentTopic) {
                            tempNonPersistentTopics.add((NonPersistentTopic) topic);
                        } else {
                            log.warn("Unsupported type of topic {}", topic.getClass().getName());
                        }
                    });
                    // end persistent topics section
                    topicStatsStream.endObject();
                    if (!tempNonPersistentTopics.isEmpty()) {
                        // start non-persistent topic
                        topicStatsStream.startObject("non-persistent");
                        tempNonPersistentTopics.forEach(topic -> {
                            try {
                                topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                            } catch (Exception e) {
                                log.error("Failed to generate topic stats for topic {}: {}", topic.getName(), e.getMessage(), e);
                            }
                        });
                        // end non-persistent topics section
                        topicStatsStream.endObject();
                    }
                    // end namespace-bundle section
                    topicStatsStream.endObject();
                });
                topicStatsStream.endObject();
                // Update metricsCollection with namespace stats
                tempMetricsCollection.add(nsStats.add(namespaceName));
            } catch (Exception e) {
                log.error("Failed to generate namespace stats for namespace {}: {}", namespaceName, e.getMessage(), e);
            }
        });
        if (clusterReplicationMetrics.isMetricsEnabled()) {
            clusterReplicationMetrics.get().forEach(clusterMetric -> tempMetricsCollection.add(clusterMetric));
            clusterReplicationMetrics.reset();
        }
        brokerOperabilityMetrics.getMetrics().forEach(brokerOperabilityMetric -> tempMetricsCollection.add(brokerOperabilityMetric));
        // json end
        topicStatsStream.endObject();
    } catch (Exception e) {
        log.error("Unable to update topic stats", e);
    }
    // swap metricsCollection and tempMetricsCollection
    List<Metrics> tempRefMetrics = metricsCollection;
    metricsCollection = tempMetricsCollection;
    tempMetricsCollection = tempRefMetrics;
    bufferLock.writeLock().lock();
    try {
        ByteBuf tmp = topicStatsBuf;
        topicStatsBuf = tempTopicStatsBuf;
        tempTopicStatsBuf = tmp;
        tempTopicStatsBuf.clear();
    } finally {
        bufferLock.writeLock().unlock();
    }
}

Example 3 with Metrics

use of org.apache.pulsar.common.stats.Metrics in project incubator-pulsar by apache.

the class JvmMetrics method generate.

public Metrics generate() {
    Map<String, String> dimensionMap = Maps.newHashMap();
    dimensionMap.put("system", "jvm");
    Metrics m = create(dimensionMap);
    Runtime r = Runtime.getRuntime();
    m.put("jvm_heap_used", r.totalMemory() - r.freeMemory());
    m.put("jvm_max_memory", r.maxMemory());
    m.put("jvm_total_memory", r.totalMemory());
    m.put("jvm_max_direct_memory", PlatformDependent.maxDirectMemory());
    m.put("jvm_thread_cnt", getThreadCount());
    m.put("jvm_gc_young_pause", currentYoungGcTime);
    m.put("jvm_gc_young_count", currentYoungGcCount);
    m.put("jvm_gc_old_pause", currentOldGcTime);
    m.put("jvm_gc_old_count", currentOldGcCount);
    long totalAllocated = 0;
    long totalUsed = 0;
    for (PoolArenaMetric arena : PooledByteBufAllocator.DEFAULT.metric().directArenas()) {
        for (PoolChunkListMetric list : arena.chunkLists()) {
            for (PoolChunkMetric chunk : list) {
                int size = chunk.chunkSize();
                int used = size - chunk.freeBytes();
                totalAllocated += size;
                totalUsed += used;
            }
        }
    }
    m.put("proxy_default_pool_allocated", totalAllocated);
    m.put("proxy_default_pool_used", totalUsed);
    return m;
}

Example 4 with Metrics

use of org.apache.pulsar.common.stats.Metrics in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testBrokerTopicStats.

@Test
public void testBrokerTopicStats() throws Exception {
    BrokerService brokerService = this.pulsar.getBrokerService();
    Field field = BrokerService.class.getDeclaredField("statsUpdater");
    field.setAccessible(true);
    ScheduledExecutorService statsUpdater = (ScheduledExecutorService) field.get(brokerService);
    // disable statsUpdate to calculate rates explicitly
    statsUpdater.shutdown();
    final String namespace = "prop/use/ns-abc";
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://" + namespace + "/topic0").create();
    // 1. producer publish messages
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Metrics metric = null;
    // sleep 1 sec to caclulate metrics per second
    Thread.sleep(1000);
    brokerService.updateRates();
    List<Metrics> metrics = brokerService.getTopicMetrics();
    for (int i = 0; i < metrics.size(); i++) {
        if (metrics.get(i).getDimension("namespace").equalsIgnoreCase(namespace)) {
            metric = metrics.get(i);
            break;
        }
    }
    assertNotNull(metric);
    double msgInRate = (double) metrics.get(0).getMetrics().get("brk_in_rate");
    // rate should be calculated and no must be > 0 as we have produced 10 msgs so far
    assertTrue(msgInRate > 0);
}

Example 5 with Metrics

use of org.apache.pulsar.common.stats.Metrics in project incubator-pulsar by apache.

the class ManagedLedgerMetricsTest method testManagedLedgerMetrics.

@Test
public void testManagedLedgerMetrics() throws Exception {
    ManagedLedgerMetrics metrics = new ManagedLedgerMetrics(pulsar);
    final String addEntryRateKey = "brk_ml_AddEntryMessagesRate";
    List<Metrics> list1 = metrics.generate();
    Assert.assertTrue(list1.isEmpty());
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (Entry<String, ManagedLedgerImpl> ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().entrySet()) {
        ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats();
        stats.refreshStats(1, TimeUnit.SECONDS);
    }
    List<Metrics> list2 = metrics.generate();
    Assert.assertEquals(list2.get(0).getMetrics().get(addEntryRateKey), 10.0D);
    for (int i = 0; i < 5; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (Entry<String, ManagedLedgerImpl> ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().entrySet()) {
        ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats();
        stats.refreshStats(1, TimeUnit.SECONDS);
    }
    List<Metrics> list3 = metrics.generate();
    Assert.assertEquals(list3.get(0).getMetrics().get(addEntryRateKey), 5.0D);
}