Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 56 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithoutPartition.

@Test
public void testSimpleConsumerEventsWithoutPartition() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/failover-topic1";
    final String subName = "sub1";
    final int numMsgs = 100;
    ConsumerConfiguration consumerConf1 = new ConsumerConfiguration();
    consumerConf1.setSubscriptionType(SubscriptionType.Failover);
    consumerConf1.setConsumerName("1");
    ConsumerConfiguration consumerConf2 = new ConsumerConfiguration();
    consumerConf2.setSubscriptionType(SubscriptionType.Failover);
    consumerConf2.setConsumerName("2");
    // 1. two consumers on the same subscription
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, consumerConf2);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(topicRef);
    assertNotNull(subRef);
    // 2. validate basic dispatcher state
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    assertEquals(subRef.getDispatcher().getType(), SubType.Failover);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    // 3. consumer1 should have all the messages while consumer2 should have no messages
    Message msg = null;
    Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    // 4. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 5. master consumer failure should resend unacked messages and new messages to another consumer
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    for (int i = 5; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
    // do not ack
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer2.acknowledge(msg);
    }
    Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 6. consumer subscription should send messages to the new consumer if its name is highest in the list
    for (int i = 0; i < 5; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer2.acknowledge(msg);
    }
    consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    Assert.assertNull(consumer1.receive(1, TimeUnit.SECONDS));
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 7. consumer subscription should not send messages to the new consumer if its name is not highest in the list
    ConsumerConfiguration consumerConf3 = new ConsumerConfiguration();
    consumerConf3.setSubscriptionType(SubscriptionType.Failover);
    consumerConf3.setConsumerName("3");
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    Consumer consumer3 = pulsarClient.subscribe(topicName, subName, consumerConf3);
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    Assert.assertNull(consumer3.receive(1, TimeUnit.SECONDS));
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 8. unsubscribe not allowed if multiple consumers connected
    try {
        consumer1.unsubscribe();
        fail("should fail");
    } catch (PulsarClientException e) {
    // ok
    }
    // 9. unsubscribe allowed if there is a lone consumer
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    consumer2.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    try {
        consumer3.unsubscribe();
    } catch (PulsarClientException e) {
        fail("Should not fail", e);
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getPersistentSubscription(subName);
    assertNull(subRef);
    producer.close();
    consumer3.close();
    admin.persistentTopics().delete(topicName);
}

Example 57 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentQueueE2ETest method testSimpleConsumerEvents.

@Test
public void testSimpleConsumerEvents() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic1";
    final String subName = "sub1";
    final int numMsgs = 100;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    // 1. two consumers on the same subscription
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(topicRef);
    assertNotNull(subRef);
    // 2. validate basic dispatcher state
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    assertEquals(subRef.getDispatcher().getType(), SubType.Shared);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs * 2);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs * 2; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs * 2);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    // both consumers will together consumer all messages
    Message msg;
    Consumer c = consumer1;
    while (true) {
        try {
            msg = c.receive(1, TimeUnit.SECONDS);
            c.acknowledge(msg);
        } catch (PulsarClientException e) {
            if (c.equals(consumer1)) {
                c = consumer2;
            } else {
                break;
            }
        }
    }
    rolloverPerIntervalStats();
    // 3. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 4. shared consumer unsubscribe not allowed
    try {
        consumer1.unsubscribe();
        fail("should fail");
    } catch (PulsarClientException e) {
    // ok
    }
    // 5. cumulative acks disabled
    consumer1.close();
    producer.send("message".getBytes());
    msg = consumer2.receive();
    try {
        consumer2.acknowledgeCumulative(msg);
        fail("Should fail");
    } catch (PulsarClientException e) {
        assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
    }
    // 6. unsubscribe allowed if this is the lone consumer
    try {
        consumer2.unsubscribe();
    } catch (PulsarClientException e) {
        fail("Should not fail");
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getPersistentSubscription(subName);
    assertNull(subRef);
    producer.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}

Example 58 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method testConcurrentTopicGCAndSubscriptionDelete.

// @Test
public void testConcurrentTopicGCAndSubscriptionDelete() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                // Thread.sleep(5,0);
                log.info("{} forcing topic GC ", Thread.currentThread());
                for (int i = 0; i < 2000; i++) {
                    topic.checkGC(0);
                }
                log.info("GC done..");
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                // do subscription delete
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                // Thread.sleep(2,0);
                log.info("unsubscriber outcome is {}", ps.doUnsubscribe(ps.getConsumers().get(0)).get());
            // assertFalse(ps.delete().isCompletedExceptionally());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}

Example 59 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method testConcurrentTopicAndSubscriptionDelete.

// @Test
public void testConcurrentTopicAndSubscriptionDelete() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                Thread.sleep(5, 0);
                log.info("deleter outcome is {}", topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                // do subscription delete
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                // Thread.sleep(2,0);
                log.info("unsubscriber outcome is {}", ps.doUnsubscribe(ps.getConsumers().get(0)).get());
            // assertFalse(ps.delete().isCompletedExceptionally());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}

Example 60 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicE2ETest method testGracefulClose.

@Test(enabled = false)
public // TODO: enable this after java client supports graceful close
void testGracefulClose() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic4";
    final String subName = "sub4";
    Producer producer = pulsarClient.createProducer(topicName);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    ExecutorService executor = Executors.newCachedThreadPool();
    CountDownLatch latch = new CountDownLatch(1);
    executor.submit(() -> {
        for (int i = 0; i < 10; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        latch.countDown();
        return null;
    });
    producer.close();
    // 1. verify there are no pending publish acks once the producer close
    // is completed on client
    assertEquals(topicRef.getProducers().values().iterator().next().getPendingPublishAcks(), 0);
    // safety latch in case of failure,
    // wait for the spawned thread to complete
    latch.await();
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(subRef);
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
    }
    // message acks
    try {
        consumer.close();
        fail("should have failed");
    } catch (IllegalStateException e) {
    // Expected - messages not acked
    }
    consumer.acknowledgeCumulative(msg);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    // 3. verify consumer close succeeds once all messages are ack'ed
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertTrue(subRef.getDispatcher().isConsumerConnected());
}