Examples with NonPersistentTopicStats org.apache.pulsar.common.policies.data.NonPersistentTopicStats used on opensource projects

Example 1 with NonPersistentTopicStats

use of org.apache.pulsar.common.policies.data.NonPersistentTopicStats in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testMsgDropStat.

/**
 * Verifies msg-drop stats
 *
 * @throws Exception
 */
@Test
public void testMsgDropStat() throws Exception {
    int defaultNonPersistentMessageRate = conf.getMaxConcurrentNonPersistentMessagePerConnection();
    try {
        final String topicName = "non-persistent://my-property/use/my-ns/stats-topic";
        // restart broker with lower publish rate limit
        conf.setMaxConcurrentNonPersistentMessagePerConnection(1);
        stopBroker();
        startBroker();
        Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-1").receiverQueueSize(1).subscribe();
        Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName("subscriber-2").receiverQueueSize(1).subscriptionType(SubscriptionType.Shared).subscribe();
        ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).create();
        String firstTimeConnected = producer.getConnectedSince();
        ExecutorService executor = Executors.newFixedThreadPool(5);
        byte[] msgData = "testData".getBytes();
        final int totalProduceMessages = 200;
        CountDownLatch latch = new CountDownLatch(totalProduceMessages);
        for (int i = 0; i < totalProduceMessages; i++) {
            executor.submit(() -> {
                producer.sendAsync(msgData).handle((msg, e) -> {
                    latch.countDown();
                    return null;
                });
            });
        }
        latch.await();
        NonPersistentTopic topic = (NonPersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
        pulsar.getBrokerService().updateRates();
        NonPersistentTopicStats stats = topic.getStats();
        NonPersistentPublisherStats npStats = stats.getPublishers().get(0);
        NonPersistentSubscriptionStats sub1Stats = stats.getSubscriptions().get("subscriber-1");
        NonPersistentSubscriptionStats sub2Stats = stats.getSubscriptions().get("subscriber-2");
        assertTrue(npStats.msgDropRate > 0);
        assertTrue(sub1Stats.msgDropRate > 0);
        assertTrue(sub2Stats.msgDropRate > 0);
        // make sure producer connection not disconnected due to unordered ack
        assertEquals(firstTimeConnected, producer.getConnectedSince());
        producer.close();
        consumer.close();
        consumer2.close();
        executor.shutdown();
    } finally {
        conf.setMaxConcurrentNonPersistentMessagePerConnection(defaultNonPersistentMessageRate);
    }
}

Example 2 with NonPersistentTopicStats

use of org.apache.pulsar.common.policies.data.NonPersistentTopicStats in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testTopicStats.

/**
 * verifies that broker is capturing topic stats correctly
 */
@Test
public void testTopicStats() throws Exception {
    final String topicName = "non-persistent://my-property/use/my-ns/unacked-topic";
    final String subName = "non-persistent";
    final int timeWaitToSync = 100;
    NonPersistentTopicStats stats;
    SubscriptionStats subStats;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared).subscriptionName(subName).subscribe();
    Thread.sleep(timeWaitToSync);
    NonPersistentTopic topicRef = (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    rolloverPerIntervalStats(pulsar);
    stats = topicRef.getStats();
    subStats = stats.getSubscriptions().values().iterator().next();
    // subscription stats
    assertEquals(stats.getSubscriptions().keySet().size(), 1);
    assertEquals(subStats.consumers.size(), 1);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Thread.sleep(timeWaitToSync);
    int totalProducedMessages = 100;
    for (int i = 0; i < totalProducedMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(timeWaitToSync);
    rolloverPerIntervalStats(pulsar);
    stats = topicRef.getStats();
    subStats = stats.getSubscriptions().values().iterator().next();
    assertTrue(subStats.msgRateOut > 0);
    assertEquals(subStats.consumers.size(), 1);
    assertTrue(subStats.msgThroughputOut > 0);
    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    assertEquals(subStats.msgRateRedeliver, 0.0);
    producer.close();
    consumer.close();
}

Example 3 with NonPersistentTopicStats

use of org.apache.pulsar.common.policies.data.NonPersistentTopicStats in project incubator-pulsar by apache.

the class NonPersistentTopic method getStats.

public NonPersistentTopicStats getStats() {
    NonPersistentTopicStats stats = new NonPersistentTopicStats();
    ObjectObjectHashMap<String, PublisherStats> remotePublishersStats = new ObjectObjectHashMap<String, PublisherStats>();
    producers.forEach(producer -> {
        NonPersistentPublisherStats publisherStats = (NonPersistentPublisherStats) producer.getStats();
        stats.msgRateIn += publisherStats.msgRateIn;
        stats.msgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        } else {
            stats.getPublishers().add(publisherStats);
        }
    });
    stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn);
    subscriptions.forEach((name, subscription) -> {
        NonPersistentSubscriptionStats subStats = subscription.getStats();
        stats.msgRateOut += subStats.msgRateOut;
        stats.msgThroughputOut += subStats.msgThroughputOut;
        stats.getSubscriptions().put(name, subStats);
    });
    replicators.forEach((cluster, replicator) -> {
        NonPersistentReplicatorStats ReplicatorStats = replicator.getStats();
        // Add incoming msg rates
        PublisherStats pubStats = remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            ReplicatorStats.msgRateIn = pubStats.msgRateIn;
            ReplicatorStats.msgThroughputIn = pubStats.msgThroughputIn;
            ReplicatorStats.inboundConnection = pubStats.address;
            ReplicatorStats.inboundConnectedSince = pubStats.connectedSince;
        }
        stats.msgRateOut += ReplicatorStats.msgRateOut;
        stats.msgThroughputOut += ReplicatorStats.msgThroughputOut;
        stats.getReplication().put(replicator.getRemoteCluster(), ReplicatorStats);
    });
    return stats;
}

Example 4 with NonPersistentTopicStats

use of org.apache.pulsar.common.policies.data.NonPersistentTopicStats in project incubator-pulsar by apache.

the class NonPersistentTopicsImpl method getStatsAsync.

@Override
public CompletableFuture<NonPersistentTopicStats> getStatsAsync(String topic) {
    TopicName topicName = validateTopic(topic);
    final CompletableFuture<NonPersistentTopicStats> future = new CompletableFuture<>();
    WebTarget path = topicPath(topicName, "stats");
    asyncGetRequest(path, new InvocationCallback<NonPersistentTopicStats>() {

        @Override
        public void completed(NonPersistentTopicStats response) {
            future.complete(response);
        }

        @Override
        public void failed(Throwable throwable) {
            future.completeExceptionally(getApiException(throwable.getCause()));
        }
    });
    return future;
}

Example 5 with NonPersistentTopicStats

use of org.apache.pulsar.common.policies.data.NonPersistentTopicStats in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testReplicator.

/**
 * verifies that non-persistent topic replicates using replicator
 */
@Test
public void testReplicator() throws Exception {
    ReplicationClusterManager replication = new ReplicationClusterManager();
    replication.setupReplicationCluster();
    try {
        final String globalTopicName = "non-persistent://pulsar/global/ns/nonPersistentTopic";
        final int timeWaitToSync = 100;
        NonPersistentTopicStats stats;
        SubscriptionStats subStats;
        PulsarClient client1 = PulsarClient.builder().serviceUrl(replication.url1.toString()).build();
        PulsarClient client2 = PulsarClient.builder().serviceUrl(replication.url2.toString()).build();
        PulsarClient client3 = PulsarClient.builder().serviceUrl(replication.url3.toString()).build();
        ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName).subscriptionName("subscriber-2").subscribe();
        ConsumerImpl<byte[]> repl2Consumer = (ConsumerImpl<byte[]>) client2.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        ConsumerImpl<byte[]> repl3Consumer = (ConsumerImpl<byte[]>) client3.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        Producer<byte[]> producer = client1.newProducer().topic(globalTopicName).create();
        Thread.sleep(timeWaitToSync);
        PulsarService replicationPulasr = replication.pulsar1;
        // Replicator for r1 -> r2,r3
        NonPersistentTopic topicRef = (NonPersistentTopic) replication.pulsar1.getBrokerService().getTopicReference(globalTopicName);
        NonPersistentReplicator replicatorR2 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r2");
        NonPersistentReplicator replicatorR3 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r3");
        assertNotNull(topicRef);
        assertNotNull(replicatorR2);
        assertNotNull(replicatorR3);
        rolloverPerIntervalStats(replicationPulasr);
        stats = topicRef.getStats();
        subStats = stats.getSubscriptions().values().iterator().next();
        // subscription stats
        assertEquals(stats.getSubscriptions().keySet().size(), 2);
        assertEquals(subStats.consumers.size(), 1);
        Thread.sleep(timeWaitToSync);
        int totalProducedMessages = 100;
        for (int i = 0; i < totalProducedMessages; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (1) consume by consumer1
        Message<?> msg = null;
        Set<String> messageSet = Sets.newHashSet();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = consumer1.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (2) consume by consumer2
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = consumer2.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (3) consume by repl2consumer
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = repl2Consumer.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (4) consume by repl3consumer
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = repl3Consumer.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        Thread.sleep(timeWaitToSync);
        rolloverPerIntervalStats(replicationPulasr);
        stats = topicRef.getStats();
        subStats = stats.getSubscriptions().values().iterator().next();
        assertTrue(subStats.msgRateOut > 0);
        assertEquals(subStats.consumers.size(), 1);
        assertTrue(subStats.msgThroughputOut > 0);
        // consumer stats
        assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
        assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
        assertEquals(subStats.msgRateRedeliver, 0.0);
        producer.close();
        consumer1.close();
        repl2Consumer.close();
        repl3Consumer.close();
        client1.close();
        client2.close();
        client3.close();
    } finally {
        replication.shutdownReplicationCluster();
    }
}