Examples with PersistentSubscription org.apache.pulsar.broker.service.persistent.PersistentSubscription used on opensource projects

Example 21 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentTopicTest method testFailoverSubscription.

@Test
public void testFailoverSubscription() throws Exception {
    PersistentTopic topic1 = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    CommandSubscribe cmd1 = CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Failover).build();
    // 1. Subscribe with non partition topic
    Future<Consumer> f1 = topic1.subscribe(serverCnx, cmd1.getSubscription(), cmd1.getConsumerId(), cmd1.getSubType(), 0, cmd1.getConsumerName(), cmd1.getDurable(), null, Collections.emptyMap(), cmd1.getReadCompacted(), InitialPosition.Latest);
    f1.get();
    // 2. Subscribe with partition topic
    PersistentTopic topic2 = new PersistentTopic(successPartitionTopicName, ledgerMock, brokerService);
    CommandSubscribe cmd2 = CommandSubscribe.newBuilder().setConsumerId(1).setConsumerName("C1").setTopic(successPartitionTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Failover).build();
    Future<Consumer> f2 = topic2.subscribe(serverCnx, cmd2.getSubscription(), cmd2.getConsumerId(), cmd2.getSubType(), 0, cmd2.getConsumerName(), cmd2.getDurable(), null, Collections.emptyMap(), cmd2.getReadCompacted(), InitialPosition.Latest);
    f2.get();
    // 3. Subscribe and create second consumer
    CommandSubscribe cmd3 = CommandSubscribe.newBuilder().setConsumerId(2).setConsumerName("C2").setTopic(successPartitionTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Failover).build();
    Future<Consumer> f3 = topic2.subscribe(serverCnx, cmd3.getSubscription(), cmd3.getConsumerId(), cmd3.getSubType(), 0, cmd3.getConsumerName(), cmd3.getDurable(), null, Collections.emptyMap(), cmd3.getReadCompacted(), InitialPosition.Latest);
    f3.get();
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerId(), 1);
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerName(), "C1");
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(1).consumerId(), 2);
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(1).consumerName(), "C2");
    // 4. Subscribe and create third duplicate consumer
    CommandSubscribe cmd4 = CommandSubscribe.newBuilder().setConsumerId(3).setConsumerName("C1").setTopic(successPartitionTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Failover).build();
    Future<Consumer> f4 = topic2.subscribe(serverCnx, cmd4.getSubscription(), cmd4.getConsumerId(), cmd4.getSubType(), 0, cmd4.getConsumerName(), cmd4.getDurable(), null, Collections.emptyMap(), cmd4.getReadCompacted(), InitialPosition.Latest);
    f4.get();
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerId(), 1);
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerName(), "C1");
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(1).consumerId(), 3);
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(1).consumerName(), "C1");
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(2).consumerId(), 2);
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(2).consumerName(), "C2");
    // 5. Subscribe on partition topic with existing consumer id and different sub type
    CommandSubscribe cmd5 = CommandSubscribe.newBuilder().setConsumerId(2).setConsumerName("C1").setTopic(successPartitionTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
    Future<Consumer> f5 = topic2.subscribe(serverCnx, cmd5.getSubscription(), cmd5.getConsumerId(), cmd5.getSubType(), 0, cmd5.getConsumerName(), cmd5.getDurable(), null, Collections.emptyMap(), cmd5.getReadCompacted(), InitialPosition.Latest);
    try {
        f5.get();
        fail("should fail with exception");
    } catch (ExecutionException ee) {
        // Expected
        assertTrue(ee.getCause() instanceof BrokerServiceException.SubscriptionBusyException);
    }
    // 6. Subscribe on partition topic with different sub name, type and different consumer id
    CommandSubscribe cmd6 = CommandSubscribe.newBuilder().setConsumerId(4).setConsumerName("C3").setTopic(successPartitionTopicName).setSubscription(successSubName2).setRequestId(1).setSubType(SubType.Exclusive).build();
    Future<Consumer> f6 = topic2.subscribe(serverCnx, cmd6.getSubscription(), cmd6.getConsumerId(), cmd6.getSubType(), 0, cmd6.getConsumerName(), cmd6.getDurable(), null, Collections.emptyMap(), cmd6.getReadCompacted(), InitialPosition.Latest);
    f6.get();
    // 7. unsubscribe exclusive sub
    Future<Void> f7 = topic2.unsubscribe(successSubName2);
    f7.get();
    assertNull(topic2.getSubscription(successSubName2));
    // 8. unsubscribe active consumer from shared sub.
    PersistentSubscription sub = topic2.getSubscription(successSubName);
    Consumer cons = sub.getDispatcher().getConsumers().get(0);
    sub.removeConsumer(cons);
    // Verify second consumer become active
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerId(), 3);
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerName(), "C1");
    // 9. unsubscribe active consumer from shared sub.
    cons = sub.getDispatcher().getConsumers().get(0);
    sub.removeConsumer(cons);
    // Verify second consumer become active
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerId(), 2);
    assertEquals(topic2.getSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerName(), "C2");
    // 10. unsubscribe shared sub
    Future<Void> f8 = topic2.unsubscribe(successSubName);
    f8.get();
    assertNull(topic2.getSubscription(successSubName));
}

Example 22 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentTopicTest method testAddRemoveConsumer.

@Test
public void testAddRemoveConsumer() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock);
    // 1. simple add consumer
    Consumer consumer = new Consumer(sub, SubType.Exclusive, topic.getName(), 1, /* consumer id */
    0, "Cons1", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest);
    sub.addConsumer(consumer);
    assertTrue(sub.getDispatcher().isConsumerConnected());
    // 2. duplicate add consumer
    try {
        sub.addConsumer(consumer);
        fail("Should fail with ConsumerBusyException");
    } catch (BrokerServiceException e) {
        assertTrue(e instanceof BrokerServiceException.ConsumerBusyException);
    }
    // 3. simple remove consumer
    sub.removeConsumer(consumer);
    assertFalse(sub.getDispatcher().isConsumerConnected());
    // 4. duplicate remove consumer
    try {
        sub.removeConsumer(consumer);
        fail("Should fail with ServerMetadataException");
    } catch (BrokerServiceException e) {
        assertTrue(e instanceof BrokerServiceException.ServerMetadataException);
    }
}

Example 23 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class SubscriptionSeekTest method testSeek.

@Test
public void testSeek() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testSeek";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    // Disable pre-fetch in consumer to track the messages received
    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscription").receiverQueueSize(0).subscribe();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    assertEquals(topicRef.getSubscriptions().size(), 1);
    List<MessageId> messageIds = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        MessageId msgId = producer.send(message.getBytes());
        messageIds.add(msgId);
    }
    PersistentSubscription sub = topicRef.getSubscription("my-subscription");
    assertEquals(sub.getNumberOfEntriesInBacklog(), 10);
    consumer.seek(MessageId.latest);
    assertEquals(sub.getNumberOfEntriesInBacklog(), 0);
    // Wait for consumer to reconnect
    Thread.sleep(500);
    consumer.seek(MessageId.earliest);
    assertEquals(sub.getNumberOfEntriesInBacklog(), 10);
    Thread.sleep(500);
    consumer.seek(messageIds.get(5));
    assertEquals(sub.getNumberOfEntriesInBacklog(), 5);
}

Example 24 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testConsumerGroupChangesWithOldNewConsumers.

@Test
public void testConsumerGroupChangesWithOldNewConsumers() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock);
    int partitionIndex = 0;
    PersistentDispatcherSingleActiveConsumer pdfc = new PersistentDispatcherSingleActiveConsumer(cursorMock, SubType.Failover, partitionIndex, topic);
    // 1. Verify no consumers connected
    assertFalse(pdfc.isConsumerConnected());
    // 2. Add old consumer
    Consumer consumer1 = new Consumer(sub, SubType.Exclusive, topic.getName(), 1, /* consumer id */
    0, "Cons1", /* consumer name */
    50000, serverCnxWithOldVersion, "myrole-1", Collections.emptyMap(), false, InitialPosition.Latest);
    pdfc.addConsumer(consumer1);
    List<Consumer> consumers = pdfc.getConsumers();
    assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
    assertEquals(1, consumers.size());
    assertNull(consumerChanges.poll());
    verify(channelCtx, times(0)).write(any());
    // 3. Add new consumer
    Consumer consumer2 = new Consumer(sub, SubType.Exclusive, topic.getName(), 2, /* consumer id */
    0, "Cons2", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, InitialPosition.Latest);
    pdfc.addConsumer(consumer2);
    consumers = pdfc.getConsumers();
    assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
    assertEquals(2, consumers.size());
    CommandActiveConsumerChange change = consumerChanges.take();
    verifyActiveConsumerChange(change, 2, false);
    verify(channelCtx, times(1)).writeAndFlush(any(), any());
}

Example 25 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentFailoverE2ETest method testActiveConsumerFailoverWithDelay.

@Test
public void testActiveConsumerFailoverWithDelay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/failover-topic3";
    final String subName = "sub1";
    final int numMsgs = 100;
    List<Message<byte[]>> receivedMessages = Lists.newArrayList();
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Failover).messageListener((consumer, msg) -> {
        try {
            synchronized (receivedMessages) {
                receivedMessages.add(msg);
            }
            consumer.acknowledge(msg);
        } catch (Exception e) {
            fail("Should not fail");
        }
    });
    ConsumerBuilder<byte[]> consumerBuilder1 = consumerBuilder.clone().consumerName("1");
    ConsumerBuilder<byte[]> consumerBuilder2 = consumerBuilder.clone().consumerName("2");
    conf.setActiveConsumerFailoverDelayTimeMillis(500);
    restartBroker();
    // create subscription
    Consumer<byte[]> consumer = consumerBuilder1.subscribe();
    consumer.close();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    // enqueue messages
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    producer.close();
    // two consumers subscribe at almost the same time
    CompletableFuture<Consumer<byte[]>> subscribeFuture2 = consumerBuilder2.subscribeAsync();
    CompletableFuture<Consumer<byte[]>> subscribeFuture1 = consumerBuilder1.subscribeAsync();
    // wait for all messages to be dequeued
    int retry = 20;
    for (int i = 0; i < retry; i++) {
        if (receivedMessages.size() >= numMsgs && subRef.getNumberOfEntriesInBacklog() == 0) {
            break;
        } else if (i != retry - 1) {
            Thread.sleep(100);
        }
    }
    // check if message duplication has occurred
    assertEquals(receivedMessages.size(), numMsgs);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < receivedMessages.size(); i++) {
        Assert.assertNotNull(receivedMessages.get(i));
        Assert.assertEquals(new String(receivedMessages.get(i).getData()), "my-message-" + i);
    }
    subscribeFuture1.get().close();
    subscribeFuture2.get().unsubscribe();
    admin.persistentTopics().delete(topicName);
    resetConfig();
    restartBroker();
}