Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 66 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testSimpleCloseTopic.

@Test
public void testSimpleCloseTopic() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic5";
    final String subName = "sub5";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(subRef);
    Message msg;
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    producer.close();
    consumer.close();
    topicRef.close().get();
    assertNull(pulsar.getBrokerService().getTopicReference(topicName));
}

Example 67 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testProducerReturnedMessageId.

@Test
public void testProducerReturnedMessageId() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-xyz";
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    long ledgerId = managedLedger.getLedgersInfoAsList().get(0).getLedgerId();
    // 2. producer publish messages
    final int SyncMessages = 10;
    for (int i = 0; i < SyncMessages; i++) {
        String message = "my-message-" + i;
        MessageId receivedMessageId = producer.send(message.getBytes());
        assertEquals(receivedMessageId, new MessageIdImpl(ledgerId, i, -1));
    }
    // 3. producer publish messages async
    final int AsyncMessages = 10;
    final CountDownLatch counter = new CountDownLatch(AsyncMessages);
    for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
        String content = "my-message-" + i;
        Message msg = MessageBuilder.create().setContent(content.getBytes()).build();
        final int index = i;
        producer.sendAsync(msg).thenRun(() -> {
            assertEquals(msg.getMessageId(), new MessageIdImpl(ledgerId, index, -1));
            counter.countDown();
        }).exceptionally((ex) -> {
            return null;
        });
    }
    counter.await();
    // 4. producer disconnect
    producer.close();
}

Example 68 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testNonBatchCumulativeAckAfterBatchPublish.

@Test
public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception {
    int numMsgs = 10;
    int numMsgsInBatch = numMsgs;
    final String topicName = "persistent://prop/use/ns-abc/testNonBatchCumulativeAckAfterBatchPublish";
    final String subscriptionName = "nbcaabp-sub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    // create producer to publish non batch messages
    Producer noBatchProducer = pulsarClient.createProducer(topicName);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    sendFutureList.clear();
    byte[] nobatchmsg = ("nobatch").getBytes();
    Message nmsg = MessageBuilder.create().setContent(nobatchmsg).build();
    noBatchProducer.sendAsync(nmsg).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i <= numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        lastunackedMsg = msg;
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    rolloverPerIntervalStats();
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    assertTrue(((ConsumerImpl) consumer).isBatchingAckTrackerEmpty());
    consumer.close();
    producer.close();
    noBatchProducer.close();
}

Example 69 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchSyncProducerWithFixedBatchSize.

@Test
public void testSimpleBatchSyncProducerWithFixedBatchSize() throws Exception {
    int numMsgs = 10;
    int numMsgsInBatch = numMsgs / 2;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchSyncProducerWithFixedBatchSize";
    final String subscriptionName = "syncsub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(1000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        producer.send(msg);
    }
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    // we expect 10 messages in the backlog since we sent 10 messages with the batch size set to 5.
    // However, we are using synchronous send and so each message will go as an individual message
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 10);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        String receivedMessage = new String(msg.getData());
        String expectedMessage = "my-message-" + i;
        Assert.assertEquals(receivedMessage, expectedMessage, "Received message " + receivedMessage + " did not match the expected message " + expectedMessage);
    }
    consumer.close();
    producer.close();
}

Example 70 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testBatchProducerWithLargeMessage.

@Test(dataProvider = "codec")
public void testBatchProducerWithLargeMessage(CompressionType compressionType) throws Exception {
    int numMsgs = 50;
    int numMsgsInBatch = numMsgs / 2;
    final String topicName = "persistent://prop/use/finance/testBatchProducerWithLargeMessage";
    final String subscriptionName = "large-message-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        if (i == 25) {
            // send a large message
            byte[] largeMessage = new byte[128 * 1024 + 4];
            Message msg = MessageBuilder.create().setContent(largeMessage).build();
            sendFutureList.add(producer.sendAsync(msg));
        } else {
            byte[] message = ("msg-" + i).getBytes();
            Message msg = MessageBuilder.create().setContent(message).build();
            sendFutureList.add(producer.sendAsync(msg));
        }
    }
    byte[] message = ("msg-" + "last").getBytes();
    Message lastMsg = MessageBuilder.create().setContent(message).build();
    sendFutureList.add(producer.sendAsync(lastMsg));
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    // we expect 3 messages in the backlog since the large message in the middle should
    // close out the batch and be sent in a batch of its own
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 3);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    for (int i = 0; i <= numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        LOG.info("received msg - {}", msg.getData().toString());
        consumer.acknowledge(msg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}