Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 51 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testOutOfOrderAcksForBatchMessage.

// test for ack holes
/*
     * lid eid bid 0 0 1-10 ack type cumul till id 9 0 1 1-10 ack type cumul on batch id 5. (should remove 0,1, 10 also
     * on broker) individual ack on 6-10. (if ack type individual on bid 5, then hole remains which is ok) 0 2 1-10 0 3
     * 1-10
     */
@Test
public void testOutOfOrderAcksForBatchMessage() throws Exception {
    int numMsgs = 40;
    int numMsgsInBatch = numMsgs / 4;
    final String topicName = "persistent://prop/use/ns-abc/testOutOfOrderAcksForBatchMessage";
    final String subscriptionName = "oooack-sub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Set<Integer> individualAcks = new HashSet<>();
    for (int i = 15; i < 20; i++) {
        individualAcks.add(i);
    }
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        LOG.info("received message {}", String.valueOf(msg.getData()));
        assertNotNull(msg);
        if (i == 8) {
            consumer.acknowledgeCumulative(msg);
        } else if (i == 9) {
        // do not ack
        } else if (i == 14) {
            // should ack lid =0 eid = 1 on broker
            consumer.acknowledgeCumulative(msg);
            Thread.sleep(1000);
            rolloverPerIntervalStats();
            Thread.sleep(1000);
            assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 3);
        } else if (individualAcks.contains(i)) {
            consumer.acknowledge(msg);
        } else {
            lastunackedMsg = msg;
        }
    }
    Thread.sleep(1000);
    rolloverPerIntervalStats();
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}

Example 52 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BrokerServiceTest method testBrokerServicePersistentTopicStats.

@Test
public void testBrokerServicePersistentTopicStats() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/successTopic";
    final String subName = "successSub";
    PersistentTopicStats stats;
    PersistentSubscriptionStats subStats;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);
    Producer producer = pulsarClient.createProducer(topicName);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);
    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);
    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    Message msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    assertEquals(subStats.msgBacklog, 0);
}

Example 53 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BrokerServiceTest method testBrokerServicePersistentRedeliverTopicStats.

@Test
public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/successSharedTopic";
    final String subName = "successSharedSub";
    PersistentTopicStats stats;
    PersistentSubscriptionStats subStats;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);
    Producer producer = pulsarClient.createProducer(topicName);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);
    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);
    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    assertEquals(subStats.msgRateRedeliver, 0.0);
    assertEquals(subStats.consumers.get(0).unackedMessages, 10);
    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(subStats.unackedMessages, subStats.consumers.get(0).unackedMessages);
    consumer.redeliverUnacknowledgedMessages();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    assertTrue(subStats.msgRateRedeliver > 0.0);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    Message msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    assertEquals(subStats.msgBacklog, 0);
}

Example 54 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentDispatcherFailoverConsumerTest method testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel.

@Test
public void testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock);
    Consumer consumer1 = createConsumer(0, 2, 1);
    Consumer consumer2 = createConsumer(0, 2, 2);
    Consumer consumer3 = createConsumer(0, 2, 3);
    Consumer consumer4 = createConsumer(1, 2, 4);
    Consumer consumer5 = createConsumer(1, 1, 5);
    Consumer consumer6 = createConsumer(1, 2, 6);
    Consumer consumer7 = createConsumer(2, 1, 7);
    Consumer consumer8 = createConsumer(2, 1, 8);
    Consumer consumer9 = createConsumer(2, 1, 9);
    dispatcher.addConsumer(consumer1);
    dispatcher.addConsumer(consumer2);
    dispatcher.addConsumer(consumer3);
    dispatcher.addConsumer(consumer4);
    dispatcher.addConsumer(consumer5);
    dispatcher.addConsumer(consumer6);
    dispatcher.addConsumer(consumer7);
    dispatcher.addConsumer(consumer8);
    dispatcher.addConsumer(consumer9);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer5);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer8);
    // in between add upper priority consumer with more permits
    Consumer consumer10 = createConsumer(0, 2, 10);
    dispatcher.addConsumer(consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer9);
}

Example 55 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithPartition.

@Test(enabled = false)
public void testSimpleConsumerEventsWithPartition() throws Exception {
    int numPartitions = 4;
    final String topicName = "persistent://prop/use/ns-abc/failover-topic2";
    final DestinationName destName = DestinationName.get(topicName);
    final String subName = "sub1";
    final int numMsgs = 100;
    Set<String> uniqueMessages = new HashSet<>();
    admin.persistentTopics().createPartitionedTopic(topicName, numPartitions);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    ConsumerConfiguration consumerConf1 = new ConsumerConfiguration();
    consumerConf1.setSubscriptionType(SubscriptionType.Failover);
    consumerConf1.setConsumerName("1");
    ConsumerConfiguration consumerConf2 = new ConsumerConfiguration();
    consumerConf2.setSubscriptionType(SubscriptionType.Failover);
    consumerConf2.setConsumerName("2");
    // 1. two consumers on the same subscription
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, consumerConf2);
    PersistentTopic topicRef;
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(0).toString());
    PersistentDispatcherSingleActiveConsumer disp0 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(1).toString());
    PersistentDispatcherSingleActiveConsumer disp1 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(2).toString());
    PersistentDispatcherSingleActiveConsumer disp2 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(3).toString());
    PersistentDispatcherSingleActiveConsumer disp3 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // equal distribution between both consumers
    int totalMessages = 0;
    Message msg = null;
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer1.acknowledge(msg);
    }
    Assert.assertEquals(totalMessages, numMsgs / 2);
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer2.acknowledge(msg);
    }
    Assert.assertEquals(totalMessages, numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    totalMessages = 0;
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // add a consumer
    ConsumerConfiguration consumerConf3 = new ConsumerConfiguration();
    consumerConf3.setSubscriptionType(SubscriptionType.Failover);
    consumerConf3.setConsumerName("3");
    for (int i = 0; i < 20; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    Consumer consumer3 = pulsarClient.subscribe(topicName, subName, consumerConf3);
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    int consumer1Messages = 0;
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer1Messages, 55);
            break;
        }
        consumer1Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    int consumer2Messages = 0;
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer2Messages, 50);
            break;
        }
        consumer2Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    int consumer3Messages = 0;
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer3Messages, 15, 10);
            break;
        }
        consumer3Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    uniqueMessages.clear();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // remove a consumer
    for (int i = 0; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    consumer2Messages = 0;
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer2Messages, 70, 5);
            break;
        }
        consumer2Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    consumer3Messages = 0;
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer3Messages, 70, 5);
            break;
        }
        consumer3Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    producer.close();
    consumer2.close();
    consumer3.unsubscribe();
    admin.persistentTopics().deletePartitionedTopic(topicName);
}