Examples with PersistentTopic org.apache.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 31 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class ServerCnxTest method testProducerFailureOnEncryptionRequiredTopic.

@Test(timeOut = 30000)
public void testProducerFailureOnEncryptionRequiredTopic() throws Exception {
    resetChannel();
    setChannelConnected();
    // Set encryption_required to true
    ZooKeeperDataCache<Policies> zkDataCache = mock(ZooKeeperDataCache.class);
    Policies policies = mock(Policies.class);
    policies.encryption_required = true;
    policies.clusterDispatchRate = Maps.newHashMap();
    doReturn(Optional.of(policies)).when(zkDataCache).get(AdminResource.path(POLICIES, TopicName.get(encryptionRequiredTopicName).getNamespace()));
    doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(zkDataCache).getAsync(AdminResource.path(POLICIES, TopicName.get(encryptionRequiredTopicName).getNamespace()));
    doReturn(zkDataCache).when(configCacheService).policiesCache();
    // test failure case: unencrypted producer cannot connect
    ByteBuf clientCommand = Commands.newProducer(encryptionRequiredTopicName, 2, /* producer id */
    2, /* request id */
    "unencrypted-producer", false, null);
    channel.writeInbound(clientCommand);
    Object response = getResponse();
    assertEquals(response.getClass(), CommandError.class);
    CommandError errorResponse = (CommandError) response;
    assertEquals(errorResponse.getError(), ServerError.MetadataError);
    PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(encryptionRequiredTopicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 0);
    channel.finish();
}

Example 32 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testFewBlockedConsumerSamePriority.

@Test
public void testFewBlockedConsumerSamePriority() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock);
    Consumer consumer1 = createConsumer(0, 2, false, 1);
    Consumer consumer2 = createConsumer(0, 2, false, 2);
    Consumer consumer3 = createConsumer(0, 2, false, 3);
    Consumer consumer4 = createConsumer(0, 2, false, 4);
    Consumer consumer5 = createConsumer(0, 1, true, 5);
    Consumer consumer6 = createConsumer(0, 2, true, 6);
    dispatcher.addConsumer(consumer1);
    dispatcher.addConsumer(consumer2);
    dispatcher.addConsumer(consumer3);
    dispatcher.addConsumer(consumer4);
    dispatcher.addConsumer(consumer5);
    dispatcher.addConsumer(consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), null);
}

Example 33 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testAddRemoveConsumer.

@Test
public void testAddRemoveConsumer() throws Exception {
    log.info("--- Starting PersistentDispatcherFailoverConsumerTest::testAddConsumer ---");
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock);
    int partitionIndex = 0;
    PersistentDispatcherSingleActiveConsumer pdfc = new PersistentDispatcherSingleActiveConsumer(cursorMock, SubType.Failover, partitionIndex, topic);
    // 1. Verify no consumers connected
    assertFalse(pdfc.isConsumerConnected());
    // 2. Add consumer
    Consumer consumer1 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 1, /* consumer id */
    0, "Cons1", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest));
    pdfc.addConsumer(consumer1);
    List<Consumer> consumers = pdfc.getConsumers();
    assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
    assertEquals(1, consumers.size());
    CommandActiveConsumerChange change = consumerChanges.take();
    verifyActiveConsumerChange(change, 1, true);
    verify(consumer1, times(1)).notifyActiveConsumerChange(same(consumer1));
    // 3. Add again, duplicate allowed
    pdfc.addConsumer(consumer1);
    consumers = pdfc.getConsumers();
    assertTrue(consumers.get(0).consumerName() == consumer1.consumerName());
    assertEquals(2, consumers.size());
    // 4. Verify active consumer
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
    // get the notified with who is the leader
    change = consumerChanges.take();
    verifyActiveConsumerChange(change, 1, true);
    verify(consumer1, times(2)).notifyActiveConsumerChange(same(consumer1));
    // 5. Add another consumer which does not change active consumer
    Consumer consumer2 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 2, /* consumer id */
    0, "Cons2", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest));
    pdfc.addConsumer(consumer2);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
    assertEquals(3, consumers.size());
    // get notified with who is the leader
    change = consumerChanges.take();
    verifyActiveConsumerChange(change, 2, false);
    verify(consumer1, times(2)).notifyActiveConsumerChange(same(consumer1));
    verify(consumer2, times(1)).notifyActiveConsumerChange(same(consumer1));
    // 6. Add a consumer which changes active consumer
    Consumer consumer0 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 0, /* consumer id */
    0, "Cons0", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest));
    pdfc.addConsumer(consumer0);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer0.consumerName());
    assertEquals(4, consumers.size());
    // all consumers will receive notifications
    change = consumerChanges.take();
    verifyActiveConsumerChange(change, 0, true);
    change = consumerChanges.take();
    verifyActiveConsumerChange(change, 1, false);
    change = consumerChanges.take();
    verifyActiveConsumerChange(change, 1, false);
    change = consumerChanges.take();
    verifyActiveConsumerChange(change, 2, false);
    verify(consumer0, times(1)).notifyActiveConsumerChange(same(consumer0));
    verify(consumer1, times(2)).notifyActiveConsumerChange(same(consumer1));
    verify(consumer1, times(2)).notifyActiveConsumerChange(same(consumer0));
    verify(consumer2, times(1)).notifyActiveConsumerChange(same(consumer1));
    verify(consumer2, times(1)).notifyActiveConsumerChange(same(consumer0));
    // 7. Remove last consumer
    pdfc.removeConsumer(consumer2);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer0.consumerName());
    assertEquals(3, consumers.size());
    // not consumer group changes
    assertNull(consumerChanges.poll());
    // 8. Verify if we cannot unsubscribe when more than one consumer is connected
    assertFalse(pdfc.canUnsubscribe(consumer0));
    // 9. Remove active consumer
    pdfc.removeConsumer(consumer0);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
    assertEquals(2, consumers.size());
    // the remaining consumers will receive notifications
    change = consumerChanges.take();
    verifyActiveConsumerChange(change, 1, true);
    change = consumerChanges.take();
    verifyActiveConsumerChange(change, 1, true);
    // 10. Attempt to remove already removed consumer
    String cause = "";
    try {
        pdfc.removeConsumer(consumer0);
    } catch (Exception e) {
        cause = e.getMessage();
    }
    assertEquals(cause, "Consumer was not connected");
    // 11. Remove active consumer
    pdfc.removeConsumer(consumer1);
    consumers = pdfc.getConsumers();
    assertTrue(pdfc.getActiveConsumer().consumerName() == consumer1.consumerName());
    assertEquals(1, consumers.size());
    // not consumer group changes
    assertNull(consumerChanges.poll());
    // 11. With only one consumer, unsubscribe is allowed
    assertTrue(pdfc.canUnsubscribe(consumer1));
}

Example 34 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithPartition.

@Test
public void testSimpleConsumerEventsWithPartition() throws Exception {
    int numPartitions = 4;
    final String topicName = "persistent://prop/use/ns-abc/failover-topic2";
    final TopicName destName = TopicName.get(topicName);
    final String subName = "sub1";
    final int numMsgs = 100;
    Set<String> uniqueMessages = new HashSet<>();
    admin.persistentTopics().createPartitionedTopic(topicName, numPartitions);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Failover);
    // 1. two consumers on the same subscription
    ActiveInactiveListenerEvent listener1 = new ActiveInactiveListenerEvent();
    ActiveInactiveListenerEvent listener2 = new ActiveInactiveListenerEvent();
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("1").consumerEventListener(listener1).subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("2").consumerEventListener(listener2).subscribe();
    PersistentTopic topicRef;
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(0).toString());
    PersistentDispatcherSingleActiveConsumer disp0 = (PersistentDispatcherSingleActiveConsumer) topicRef.getSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(1).toString());
    PersistentDispatcherSingleActiveConsumer disp1 = (PersistentDispatcherSingleActiveConsumer) topicRef.getSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(2).toString());
    PersistentDispatcherSingleActiveConsumer disp2 = (PersistentDispatcherSingleActiveConsumer) topicRef.getSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(3).toString());
    PersistentDispatcherSingleActiveConsumer disp3 = (PersistentDispatcherSingleActiveConsumer) topicRef.getSubscription(subName).getDispatcher();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // equal distribution between both consumers
    int totalMessages = 0;
    Message<byte[]> msg = null;
    Set<Integer> receivedPtns = Sets.newHashSet();
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer1.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        receivedPtns.add(msgId.getPartitionIndex());
    }
    assertTrue(Sets.difference(listener1.activePtns, receivedPtns).isEmpty());
    assertTrue(Sets.difference(listener2.inactivePtns, receivedPtns).isEmpty());
    Assert.assertEquals(totalMessages, numMsgs / 2);
    receivedPtns = Sets.newHashSet();
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer2.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        receivedPtns.add(msgId.getPartitionIndex());
    }
    assertTrue(Sets.difference(listener1.inactivePtns, receivedPtns).isEmpty());
    assertTrue(Sets.difference(listener2.activePtns, receivedPtns).isEmpty());
    Assert.assertEquals(totalMessages, numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "1");
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "2");
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "1");
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "2");
    totalMessages = 0;
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // add a consumer
    for (int i = 0; i < 20; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    Consumer<byte[]> consumer3 = consumerBuilder.clone().consumerName("3").subscribe();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    int consumer1Messages = 0;
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer1Messages, 55);
            break;
        }
        consumer1Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    int consumer2Messages = 0;
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer2Messages, 50);
            break;
        }
        consumer2Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    int consumer3Messages = 0;
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer3Messages, 15, 10);
            break;
        }
        consumer3Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "1");
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "2");
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "3");
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "1");
    uniqueMessages.clear();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // remove a consumer
    for (int i = 0; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    consumer2Messages = 0;
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer2Messages, 70, 5);
            break;
        }
        consumer2Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    consumer3Messages = 0;
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer3Messages, 70, 5);
            break;
        }
        consumer3Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "2");
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "3");
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "2");
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "3");
    producer.close();
    consumer2.close();
    consumer3.unsubscribe();
    admin.persistentTopics().deletePartitionedTopic(topicName);
}

Example 35 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testCancelReadRequestOnLastDisconnect.

@Test(timeOut = 60000)
public void testCancelReadRequestOnLastDisconnect() throws Exception {
    String key = "testCancelReadRequestOnLastDisconnect";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(1000).subscriptionType(SubscriptionType.Shared);
    Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1 = consumer1.receive();
    Message<byte[]> message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            consumer1.acknowledge(message1);
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
            consumer2.acknowledge(message2);
        }
        message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // 5. Close Consumer 1 and 2
    log.info("Consumer 1 closed");
    log.info("Consumer 2 closed");
    consumer1.close();
    consumer2.close();
    // 6. Producer produces more messages
    for (int i = totalMessages; i < 2 * totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 7. Consumer reconnects
    consumer1 = consumerBuilder.subscribe();
    // 8. Check number of messages received
    receivedConsumer1 = 0;
    message1 = consumer1.receive();
    while (message1 != null) {
        log.info("Consumer 1 Received: " + new String(message1.getData()));
        receivedConsumer1++;
        message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
    }
    log.info("Total receives by Consumer 2 = " + receivedConsumer2);
    assertEquals(receivedConsumer1, totalMessages);
}