Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 6 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageExpiryWithFewExpiredBacklog.

@Test
public void testMessageExpiryWithFewExpiredBacklog() throws Exception {
    int messageTTLSecs = 10;
    String namespaceName = "prop/use/expiry-check-1";
    admin.namespaces().createNamespace(namespaceName);
    admin.namespaces().setNamespaceMessageTTL(namespaceName, messageTTLSecs);
    final String topicName = "persistent://prop/use/expiry-check-1/topic1";
    final String subName = "sub1";
    final int numMsgs = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    pulsarClient.subscribe(topicName, subName, conf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(TimeUnit.SECONDS.toMillis(messageTTLSecs));
    runMessageExpiryCheck();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(TimeUnit.SECONDS.toMillis(messageTTLSecs / 2));
    runMessageExpiryCheck();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
}

Example 7 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testSimpleConsumerEvents.

@Test
public void testSimpleConsumerEvents() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic1";
    final String subName = "sub1";
    final int numMsgs = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    // 1. client connect
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(topicRef);
    assertNotNull(subRef);
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), 1000);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs * 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs * 2);
    // 2. messages pushed before client receive
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), 1000 - numMsgs * 2);
    Message msg = null;
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer.receive();
        // 3. in-order message delivery
        assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    // 4. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer.receive();
        if (i == numMsgs - 1) {
            consumer.acknowledgeCumulative(msg);
        }
    }
    rolloverPerIntervalStats();
    // 5. messages deleted on cumulative acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 6. consumer unsubscribe
    consumer.unsubscribe();
    // 6. consumer graceful close
    consumer.close();
    // 7. consumer unsubscribe
    try {
        consumer.unsubscribe();
        fail("Should have failed");
    } catch (PulsarClientException.AlreadyClosedException e) {
    // ok
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getPersistentSubscription(subName);
    assertNull(subRef);
    producer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
}

Example 8 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerWithFixedBatchTime.

@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressionType) throws Exception {
    int numMsgs = 100;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchTime";
    final String subscriptionName = "time-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    Random random = new Random();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        // put a random sleep from 0 to 3 ms
        Thread.sleep(random.nextInt(4));
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    LOG.info("Sent {} messages, backlog is {} messages", numMsgs, topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog());
    assertTrue(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog() < numMsgs);
    producer.close();
}

Example 9 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerConsumer1kMessages.

@Test
public void testSimpleBatchProducerConsumer1kMessages() throws Exception {
    int numMsgs = 2000;
    int numMsgsInBatch = 4;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer1kMessages";
    final String subscriptionName = "pc1k-sub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(30000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    int sendError = 0;
    for (CompletableFuture<MessageId> sendFuture : sendFutureList) {
        if (sendFuture.isCompletedExceptionally()) {
            ++sendError;
        }
    }
    if (sendError != 0) {
        LOG.warn("[{}] Error sending {} messages", subscriptionName, sendError);
        numMsgs = numMsgs - sendError;
    }
    LOG.info("[{}] sent {} messages", subscriptionName, numMsgs);
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    // allow stats to be updated..
    Thread.sleep(5000);
    LOG.info("[{}] checking backlog stats..");
    rolloverPerIntervalStats();
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(1, TimeUnit.SECONDS);
        assertNotNull(msg);
        lastunackedMsg = msg;
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}

Example 10 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testBatchAndNonBatchCumulativeAcks.

@Test
public void testBatchAndNonBatchCumulativeAcks() throws Exception {
    int numMsgs = 50;
    int numMsgsInBatch = numMsgs / 10;
    final String topicName = "persistent://prop/use/ns-abc/testBatchAndNonBatchCumulativeAcks";
    final String subscriptionName = "bnb-sub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    // create producer to publish non batch messages
    Producer noBatchProducer = pulsarClient.createProducer(topicName);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs / 2; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
        byte[] nobatchmsg = ("nobatch-" + i).getBytes();
        msg = MessageBuilder.create().setContent(nobatchmsg).build();
        sendFutureList.add(noBatchProducer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), (numMsgs / 2) / numMsgsInBatch + numMsgs / 2);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        LOG.info("[{}] got message position{} data {}", subscriptionName, msg.getMessageId(), String.valueOf(msg.getData()));
        if (i % 2 == 0) {
            lastunackedMsg = msg;
        } else {
            consumer.acknowledgeCumulative(msg);
            LOG.info("[{}] did cumulative ack on position{} ", subscriptionName, msg.getMessageId());
        }
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    assertTrue(((ConsumerImpl) consumer).isBatchingAckTrackerEmpty());
    consumer.close();
    producer.close();
    noBatchProducer.close();
}