Examples with NonPersistentTopic org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic used on opensource projects

Example 1 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class NonPersistentTopics method getStats.

@GET
@Path("{property}/{cluster}/{namespace}/{topic}/stats")
@ApiOperation(hidden = true, value = "Get the stats for the topic.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic does not exist") })
public NonPersistentTopicStats getStats(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    validateTopicName(property, cluster, namespace, encodedTopic);
    validateAdminOperationOnTopic(authoritative);
    Topic topic = getTopicReference(topicName);
    return ((NonPersistentTopic) topic).getStats();
}

Example 2 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class PulsarStats method updateStats.

public synchronized void updateStats(ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, Topic>>> topicsMap) {
    StatsOutputStream topicStatsStream = new StatsOutputStream(tempTopicStatsBuf);
    try {
        tempMetricsCollection.clear();
        bundleStats.clear();
        brokerOperabilityMetrics.reset();
        // Json begin
        topicStatsStream.startObject();
        topicsMap.forEach((namespaceName, bundles) -> {
            if (bundles.isEmpty()) {
                return;
            }
            try {
                topicStatsStream.startObject(namespaceName);
                nsStats.reset();
                bundles.forEach((bundle, topics) -> {
                    NamespaceBundleStats currentBundleStats = bundleStats.computeIfAbsent(bundle, k -> new NamespaceBundleStats());
                    currentBundleStats.reset();
                    currentBundleStats.topics = topics.size();
                    topicStatsStream.startObject(NamespaceBundle.getBundleRange(bundle));
                    tempNonPersistentTopics.clear();
                    // start persistent topic
                    topicStatsStream.startObject("persistent");
                    topics.forEach((name, topic) -> {
                        if (topic instanceof PersistentTopic) {
                            try {
                                topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                            } catch (Exception e) {
                                log.error("Failed to generate topic stats for topic {}: {}", name, e.getMessage(), e);
                            }
                            // this task: helps to activate inactive-backlog-cursors which have caught up and
                            // connected, also deactivate active-backlog-cursors which has backlog
                            ((PersistentTopic) topic).getManagedLedger().checkBackloggedCursors();
                        } else if (topic instanceof NonPersistentTopic) {
                            tempNonPersistentTopics.add((NonPersistentTopic) topic);
                        } else {
                            log.warn("Unsupported type of topic {}", topic.getClass().getName());
                        }
                    });
                    // end persistent topics section
                    topicStatsStream.endObject();
                    if (!tempNonPersistentTopics.isEmpty()) {
                        // start non-persistent topic
                        topicStatsStream.startObject("non-persistent");
                        tempNonPersistentTopics.forEach(topic -> {
                            try {
                                topic.updateRates(nsStats, currentBundleStats, topicStatsStream, clusterReplicationMetrics, namespaceName);
                            } catch (Exception e) {
                                log.error("Failed to generate topic stats for topic {}: {}", topic.getName(), e.getMessage(), e);
                            }
                        });
                        // end non-persistent topics section
                        topicStatsStream.endObject();
                    }
                    // end namespace-bundle section
                    topicStatsStream.endObject();
                });
                topicStatsStream.endObject();
                // Update metricsCollection with namespace stats
                tempMetricsCollection.add(nsStats.add(namespaceName));
            } catch (Exception e) {
                log.error("Failed to generate namespace stats for namespace {}: {}", namespaceName, e.getMessage(), e);
            }
        });
        if (clusterReplicationMetrics.isMetricsEnabled()) {
            clusterReplicationMetrics.get().forEach(clusterMetric -> tempMetricsCollection.add(clusterMetric));
            clusterReplicationMetrics.reset();
        }
        brokerOperabilityMetrics.getMetrics().forEach(brokerOperabilityMetric -> tempMetricsCollection.add(brokerOperabilityMetric));
        // json end
        topicStatsStream.endObject();
    } catch (Exception e) {
        log.error("Unable to update topic stats", e);
    }
    // swap metricsCollection and tempMetricsCollection
    List<Metrics> tempRefMetrics = metricsCollection;
    metricsCollection = tempMetricsCollection;
    tempMetricsCollection = tempRefMetrics;
    bufferLock.writeLock().lock();
    try {
        ByteBuf tmp = topicStatsBuf;
        topicStatsBuf = tempTopicStatsBuf;
        tempTopicStatsBuf = tmp;
        tempTopicStatsBuf.clear();
    } finally {
        bufferLock.writeLock().unlock();
    }
}

Example 3 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class NonPersistentTopics method getStats.

@GET
@Path("{property}/{namespace}/{topic}/stats")
@ApiOperation(value = "Get the stats for the topic.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic does not exist") })
public NonPersistentTopicStats getStats(@PathParam("property") String property, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    validateTopicName(property, namespace, encodedTopic);
    validateAdminOperationOnTopic(topicName, authoritative);
    Topic topic = getTopicReference(topicName);
    return ((NonPersistentTopic) topic).getStats();
}

Example 4 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testMsgDropStat.

/**
 * Verifies msg-drop stats
 *
 * @throws Exception
 */
@Test
public void testMsgDropStat() throws Exception {
    int defaultNonPersistentMessageRate = conf.getMaxConcurrentNonPersistentMessagePerConnection();
    try {
        final String topicName = "non-persistent://my-property/use/my-ns/stats-topic";
        // restart broker with lower publish rate limit
        conf.setMaxConcurrentNonPersistentMessagePerConnection(1);
        stopBroker();
        startBroker();
        Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1").receiverQueueSize(1).subscribe();
        Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2").receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
        ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).create();
        String firstTimeConnected = producer.getConnectedSince();
        ExecutorService executor = Executors.newFixedThreadPool(5);
        byte[] msgData = "testData".getBytes();
        final int totalProduceMessages = 200;
        CountDownLatch latch = new CountDownLatch(totalProduceMessages);
        for (int i = 0; i < totalProduceMessages; i++) {
            executor.submit(() -> {
                producer.sendAsync(msgData).handle((msg, e) -> {
                    latch.countDown();
                    return null;
                });
            });
        }
        latch.await();
        NonPersistentTopic topic = (NonPersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
        pulsar.getBrokerService().updateRates();
        NonPersistentTopicStats stats = topic.getStats();
        NonPersistentPublisherStats npStats = stats.getPublishers().get(0);
        NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1");
        NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2");
        assertTrue(npStats.msgDropRate > 0);
        assertTrue(sub1Stats.msgDropRate > 0);
        assertTrue(sub2Stats.msgDropRate > 0);
        // make sure producer connection not disconnected due to unordered ack
        assertEquals(firstTimeConnected, producer.getConnectedSince());
        producer.close();
        consumer.close();
        consumer2.close();
        executor.shutdown();
    } finally {
        conf.setMaxConcurrentNonPersistentMessagePerConnection(defaultNonPersistentMessageRate);
    }
}

Example 5 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testTopicStats.

/**
 * verifies that broker is capturing topic stats correctly
 */
@Test
public void testTopicStats() throws Exception {
    final String topicName = "non-persistent://my-property/use/my-ns/unacked-topic";
    final String subName = "non-persistent";
    final int timeWaitToSync = 100;
    NonPersistentTopicStats stats;
    SubscriptionStats subStats;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared).subscriptionName(subName).subscribe();
    Thread.sleep(timeWaitToSync);
    NonPersistentTopic topicRef = (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    rolloverPerIntervalStats(pulsar);
    stats = topicRef.getStats();
    subStats = stats.getSubscriptions().values().iterator().next();
    // subscription stats
    assertEquals(stats.getSubscriptions().keySet().size(), 1);
    assertEquals(subStats.consumers.size(), 1);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Thread.sleep(timeWaitToSync);
    int totalProducedMessages = 100;
    for (int i = 0; i < totalProducedMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(timeWaitToSync);
    rolloverPerIntervalStats(pulsar);
    stats = topicRef.getStats();
    subStats = stats.getSubscriptions().values().iterator().next();
    assertTrue(subStats.msgRateOut > 0);
    assertEquals(subStats.consumers.size(), 1);
    assertTrue(subStats.msgThroughputOut > 0);
    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    assertEquals(subStats.msgRateRedeliver, 0.0);
    producer.close();
    consumer.close();
}