Examples with PersistentTopic org.apache.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 56 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testRateLimitingMultipleConsumers.

/**
 * verify message-rate on multiple consumers with shared-subscription
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testRateLimitingMultipleConsumers() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingMultipleConsumers";
    final int messageRate = 5;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    final int numProducedMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    });
    Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer3 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer4 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer5 = consumerBuilder.subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numProducedMessages; i++) {
        final String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // it can make sure that consumer had enough time to consume message but couldn't consume due to throttling
    Thread.sleep(500);
    // consumer should not have received all published message due to message-rate throttling
    Assert.assertNotEquals(totalReceived.get(), numProducedMessages);
    consumer1.close();
    consumer2.close();
    consumer3.close();
    consumer4.close();
    consumer5.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 57 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testMessageRateDynamicallyChange.

/**
 * verifies: message-rate change gets reflected immediately into topic at runtime
 *
 * @throws Exception
 */
@Test
public void testMessageRateDynamicallyChange() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    admin.namespaces().createNamespace(namespace);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    // (1) verify message-rate is -1 initially
    Assert.assertEquals(topic.getDispatchRateLimiter().getDispatchRateOnMsg(), -1);
    // (1) change to 100
    int messageRate = 100;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    boolean isDispatchRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0) {
            isDispatchRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isDispatchRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    // (1) change to 500
    messageRate = 500;
    dispatchRate = new DispatchRate(-1, messageRate, 360);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    isDispatchRateUpdate = false;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnByte() == messageRate) {
            isDispatchRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isDispatchRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    producer.close();
}

Example 58 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class BatchMessageTest method testNonBatchCumulativeAckAfterBatchPublish.

@Test
public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception {
    int numMsgs = 10;
    int numMsgsInBatch = numMsgs;
    final String topicName = "persistent://prop/use/ns-abc/testNonBatchCumulativeAckAfterBatchPublish";
    final String subscriptionName = "nbcaabp-sub-1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
    consumer.close();
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true).create();
    // create producer to publish non batch messages
    Producer<byte[]> noBatchProducer = pulsarClient.newProducer().topic(topicName).create();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    sendFutureList.clear();
    byte[] nobatchmsg = ("nobatch").getBytes();
    Message<byte[]> nmsg = MessageBuilder.create().setContent(nobatchmsg).build();
    noBatchProducer.sendAsync(nmsg).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
    consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
    Message<byte[]> lastunackedMsg = null;
    for (int i = 0; i <= numMsgs; i++) {
        Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        lastunackedMsg = msg;
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    rolloverPerIntervalStats();
    assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
    noBatchProducer.close();
}

Example 59 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class BatchMessageTest method testSimpleBatchSyncProducerWithFixedBatchSize.

@Test
public void testSimpleBatchSyncProducerWithFixedBatchSize() throws Exception {
    int numMsgs = 10;
    int numMsgsInBatch = numMsgs / 2;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchSyncProducerWithFixedBatchSize";
    final String subscriptionName = "syncsub-1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
    consumer.close();
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).batchingMaxPublishDelay(1, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true).create();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
        producer.send(msg);
    }
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    // we expect 10 messages in the backlog since we sent 10 messages with the batch size set to 5.
    // However, we are using synchronous send and so each message will go as an individual message
    assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 10);
    consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
    for (int i = 0; i < numMsgs; i++) {
        Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        String receivedMessage = new String(msg.getData());
        String expectedMessage = "my-message-" + i;
        Assert.assertEquals(receivedMessage, expectedMessage, "Received message " + receivedMessage + " did not match the expected message " + expectedMessage);
    }
    consumer.close();
    producer.close();
}

Example 60 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class BatchMessageTest method testOutOfOrderAcksForBatchMessage.

// test for ack holes
/*
     * lid eid bid 0 0 1-10 ack type cumul till id 9 0 1 1-10 ack type cumul on batch id 5. (should remove 0,1, 10 also
     * on broker) individual ack on 6-10. (if ack type individual on bid 5, then hole remains which is ok) 0 2 1-10 0 3
     * 1-10
     */
@Test
public void testOutOfOrderAcksForBatchMessage() throws Exception {
    int numMsgs = 40;
    int numMsgsInBatch = numMsgs / 4;
    final String topicName = "persistent://prop/use/ns-abc/testOutOfOrderAcksForBatchMessage";
    final String subscriptionName = "oooack-sub-1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
    consumer.close();
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true).create();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
    consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
    Set<Integer> individualAcks = new HashSet<>();
    for (int i = 15; i < 20; i++) {
        individualAcks.add(i);
    }
    Message<byte[]> lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
        LOG.info("received message {}", new String(msg.getData(), UTF_8));
        assertNotNull(msg);
        if (i == 8) {
            consumer.acknowledgeCumulative(msg);
        } else if (i == 9) {
        // do not ack
        } else if (i == 14) {
            // should ack lid =0 eid = 1 on broker
            consumer.acknowledgeCumulative(msg);
            Thread.sleep(1000);
            rolloverPerIntervalStats();
            Thread.sleep(1000);
            assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 3);
        } else if (individualAcks.contains(i)) {
            consumer.acknowledge(msg);
        } else {
            lastunackedMsg = msg;
        }
    }
    Thread.sleep(1000);
    rolloverPerIntervalStats();
    assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}