Examples with PersistentTopic org.apache.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 86 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testConsumerGroupChangesWithOldNewConsumers.

@Test
public void testConsumerGroupChangesWithOldNewConsumers() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock);
    int partitionIndex = 0;
    PersistentDispatcherSingleActiveConsumer pdfc = new PersistentDispatcherSingleActiveConsumer(cursorMock, SubType.Failover, partitionIndex, topic);
    // 1. Verify no consumers connected
    assertFalse(pdfc.isConsumerConnected());
    // 2. Add old consumer
    Consumer consumer1 = new Consumer(sub, SubType.Exclusive, topic.getName(), 1, /* consumer id */
    0, "Cons1", /* consumer name */
    50000, serverCnxWithOldVersion, "myrole-1", Collections.emptyMap(), false, InitialPosition.Latest);
    pdfc.addConsumer(consumer1);
    List<Consumer> consumers = pdfc.getConsumers();
    assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
    assertEquals(1, consumers.size());
    assertNull(consumerChanges.poll());
    verify(channelCtx, times(0)).write(any());
    // 3. Add new consumer
    Consumer consumer2 = new Consumer(sub, SubType.Exclusive, topic.getName(), 2, /* consumer id */
    0, "Cons2", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, InitialPosition.Latest);
    pdfc.addConsumer(consumer2);
    consumers = pdfc.getConsumers();
    assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
    assertEquals(2, consumers.size());
    CommandActiveConsumerChange change = consumerChanges.take();
    verifyActiveConsumerChange(change, 2, false);
    verify(channelCtx, times(1)).writeAndFlush(any(), any());
}

Example 87 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testFewBlockedConsumerDifferentPriority.

@Test
public void testFewBlockedConsumerDifferentPriority() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock);
    Consumer consumer1 = createConsumer(0, 2, false, 1);
    Consumer consumer2 = createConsumer(0, 2, false, 2);
    Consumer consumer3 = createConsumer(0, 2, false, 3);
    Consumer consumer4 = createConsumer(0, 2, false, 4);
    Consumer consumer5 = createConsumer(0, 1, true, 5);
    Consumer consumer6 = createConsumer(0, 2, true, 6);
    Consumer consumer7 = createConsumer(1, 2, false, 7);
    Consumer consumer8 = createConsumer(1, 10, true, 8);
    Consumer consumer9 = createConsumer(1, 2, false, 9);
    Consumer consumer10 = createConsumer(2, 2, false, 10);
    Consumer consumer11 = createConsumer(2, 10, true, 11);
    Consumer consumer12 = createConsumer(2, 2, false, 12);
    dispatcher.addConsumer(consumer1);
    dispatcher.addConsumer(consumer2);
    dispatcher.addConsumer(consumer3);
    dispatcher.addConsumer(consumer4);
    dispatcher.addConsumer(consumer5);
    dispatcher.addConsumer(consumer6);
    dispatcher.addConsumer(consumer7);
    dispatcher.addConsumer(consumer8);
    dispatcher.addConsumer(consumer9);
    dispatcher.addConsumer(consumer10);
    dispatcher.addConsumer(consumer11);
    dispatcher.addConsumer(consumer12);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer9);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer9);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer12);
    // add consumer with lower priority again
    Consumer consumer13 = createConsumer(0, 2, false, 13);
    Consumer consumer14 = createConsumer(0, 2, true, 14);
    dispatcher.addConsumer(consumer13);
    dispatcher.addConsumer(consumer14);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer13);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer13);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer12);
    Assert.assertEquals(getNextConsumer(dispatcher), null);
}

Example 88 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testFewBlockedConsumerDifferentPriority2.

@Test
public void testFewBlockedConsumerDifferentPriority2() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock);
    Consumer consumer1 = createConsumer(0, 2, true, 1);
    Consumer consumer2 = createConsumer(0, 2, true, 2);
    Consumer consumer3 = createConsumer(0, 2, true, 3);
    Consumer consumer4 = createConsumer(1, 2, false, 4);
    Consumer consumer5 = createConsumer(1, 1, false, 5);
    Consumer consumer6 = createConsumer(2, 1, false, 6);
    Consumer consumer7 = createConsumer(2, 2, true, 7);
    dispatcher.addConsumer(consumer1);
    dispatcher.addConsumer(consumer2);
    dispatcher.addConsumer(consumer3);
    dispatcher.addConsumer(consumer4);
    dispatcher.addConsumer(consumer5);
    dispatcher.addConsumer(consumer6);
    dispatcher.addConsumer(consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer5);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), null);
}

Example 89 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentFailoverE2ETest method testActiveConsumerFailoverWithDelay.

@Test
public void testActiveConsumerFailoverWithDelay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/failover-topic3";
    final String subName = "sub1";
    final int numMsgs = 100;
    List<Message<byte[]>> receivedMessages = Lists.newArrayList();
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Failover).messageListener((consumer, msg) -> {
        try {
            synchronized (receivedMessages) {
                receivedMessages.add(msg);
            }
            consumer.acknowledge(msg);
        } catch (Exception e) {
            fail("Should not fail");
        }
    });
    ConsumerBuilder<byte[]> consumerBuilder1 = consumerBuilder.clone().consumerName("1");
    ConsumerBuilder<byte[]> consumerBuilder2 = consumerBuilder.clone().consumerName("2");
    conf.setActiveConsumerFailoverDelayTimeMillis(500);
    restartBroker();
    // create subscription
    Consumer<byte[]> consumer = consumerBuilder1.subscribe();
    consumer.close();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    // enqueue messages
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    producer.close();
    // two consumers subscribe at almost the same time
    CompletableFuture<Consumer<byte[]>> subscribeFuture2 = consumerBuilder2.subscribeAsync();
    CompletableFuture<Consumer<byte[]>> subscribeFuture1 = consumerBuilder1.subscribeAsync();
    // wait for all messages to be dequeued
    int retry = 20;
    for (int i = 0; i < retry; i++) {
        if (receivedMessages.size() >= numMsgs && subRef.getNumberOfEntriesInBacklog() == 0) {
            break;
        } else if (i != retry - 1) {
            Thread.sleep(100);
        }
    }
    // check if message duplication has occurred
    assertEquals(receivedMessages.size(), numMsgs);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < receivedMessages.size(); i++) {
        Assert.assertNotNull(receivedMessages.get(i));
        Assert.assertEquals(new String(receivedMessages.get(i).getData()), "my-message-" + i);
    }
    subscribeFuture1.get().close();
    subscribeFuture2.get().unsubscribe();
    admin.persistentTopics().delete(topicName);
    resetConfig();
    restartBroker();
}

Example 90 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithoutPartition.

@Test
public void testSimpleConsumerEventsWithoutPartition() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/failover-topic1";
    final String subName = "sub1";
    final int numMsgs = 100;
    TestConsumerStateEventListener listener1 = new TestConsumerStateEventListener();
    TestConsumerStateEventListener listener2 = new TestConsumerStateEventListener();
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Failover);
    // 1. two consumers on the same subscription
    ConsumerBuilder<byte[]> consumerBulder1 = consumerBuilder.clone().consumerName("1").consumerEventListener(listener1);
    Consumer<byte[]> consumer1 = consumerBulder1.subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("2").consumerEventListener(listener2).subscribe();
    verifyConsumerActive(listener1, -1);
    verifyConsumerInactive(listener2, -1);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(topicRef);
    assertNotNull(subRef);
    // 2. validate basic dispatcher state
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    assertEquals(subRef.getDispatcher().getType(), SubType.Failover);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    // 3. consumer1 should have all the messages while consumer2 should have no messages
    Message<byte[]> msg = null;
    Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    // 4. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 5. master consumer failure should resend unacked messages and new messages to another consumer
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    for (int i = 5; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
    // do not ack
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    verifyConsumerActive(listener2, -1);
    verifyConsumerNotReceiveAnyStateChanges(listener1);
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer2.acknowledge(msg);
    }
    Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 6. consumer subscription should send messages to the new consumer if its name is highest in the list
    for (int i = 0; i < 5; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer2.acknowledge(msg);
    }
    consumer1 = consumerBulder1.subscribe();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    Assert.assertNull(consumer1.receive(1, TimeUnit.SECONDS));
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 7. consumer subscription should not send messages to the new consumer if its name is not highest in the list
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    TestConsumerStateEventListener listener3 = new TestConsumerStateEventListener();
    Consumer<byte[]> consumer3 = consumerBuilder.clone().consumerName("3").consumerEventListener(listener3).subscribe();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    verifyConsumerInactive(listener3, -1);
    Assert.assertNull(consumer3.receive(1, TimeUnit.SECONDS));
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 8. unsubscribe not allowed if multiple consumers connected
    try {
        consumer1.unsubscribe();
        fail("should fail");
    } catch (PulsarClientException e) {
    // ok
    }
    // 9. unsubscribe allowed if there is a lone consumer
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    consumer2.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    try {
        consumer3.unsubscribe();
    } catch (PulsarClientException e) {
        fail("Should not fail", e);
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getSubscription(subName);
    assertNull(subRef);
    producer.close();
    consumer3.close();
    admin.persistentTopics().delete(topicName);
}