Examples with PersistentSubscription com.yahoo.pulsar.broker.service.persistent.PersistentSubscription used on opensource projects

Example 1 with PersistentSubscription

use of com.yahoo.pulsar.broker.service.persistent.PersistentSubscription in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method testConcurrentTopicDeleteAndUnsubscribe.

// @Test
public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                Thread.sleep(4, 700);
                log.info("deleter outcome is {}", topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                // Thread.sleep(2,0);
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                log.info("unsubscribe result : {}", topic.unsubscribe(successSubName).get());
                log.info("closing consumer..");
                ps.getConsumers().get(0).close();
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}

Example 2 with PersistentSubscription

use of com.yahoo.pulsar.broker.service.persistent.PersistentSubscription in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method testConcurrentTopicDeleteAndSubsUnsubscribe.

// @Test
public void testConcurrentTopicDeleteAndSubsUnsubscribe() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                Thread.sleep(4, 730);
                log.info("@@@@@@@@ DELETER TH");
                log.info("deleter outcome is " + topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                log.info("&&&&&&&&& UNSUBSCRIBER TH");
                // Thread.sleep(2,0);
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                log.info("unsubscribe result : " + ps.doUnsubscribe(ps.getConsumers().get(0)).get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}

Example 3 with PersistentSubscription

use of com.yahoo.pulsar.broker.service.persistent.PersistentSubscription in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageExpiryWithFewExpiredBacklog.

@Test
public void testMessageExpiryWithFewExpiredBacklog() throws Exception {
    int messageTTLSecs = 10;
    String namespaceName = "prop/use/expiry-check-1";
    admin.namespaces().createNamespace(namespaceName);
    admin.namespaces().setNamespaceMessageTTL(namespaceName, messageTTLSecs);
    final String topicName = "persistent://prop/use/expiry-check-1/topic1";
    final String subName = "sub1";
    final int numMsgs = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    pulsarClient.subscribe(topicName, subName, conf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(TimeUnit.SECONDS.toMillis(messageTTLSecs));
    runMessageExpiryCheck();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(TimeUnit.SECONDS.toMillis(messageTTLSecs / 2));
    runMessageExpiryCheck();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
}

Example 4 with PersistentSubscription

use of com.yahoo.pulsar.broker.service.persistent.PersistentSubscription in project pulsar by yahoo.

the class PersistentTopicE2ETest method testSimpleConsumerEvents.

@Test
public void testSimpleConsumerEvents() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic1";
    final String subName = "sub1";
    final int numMsgs = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    // 1. client connect
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(topicRef);
    assertNotNull(subRef);
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), 1000);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs * 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs * 2);
    // 2. messages pushed before client receive
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), 1000 - numMsgs * 2);
    Message msg = null;
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer.receive();
        // 3. in-order message delivery
        assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    // 4. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer.receive();
        if (i == numMsgs - 1) {
            consumer.acknowledgeCumulative(msg);
        }
    }
    rolloverPerIntervalStats();
    // 5. messages deleted on cumulative acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 6. consumer unsubscribe
    consumer.unsubscribe();
    // 6. consumer graceful close
    consumer.close();
    // 7. consumer unsubscribe
    try {
        consumer.unsubscribe();
        fail("Should have failed");
    } catch (PulsarClientException.AlreadyClosedException e) {
    // ok
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getPersistentSubscription(subName);
    assertNull(subRef);
    producer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
}

Example 5 with PersistentSubscription

use of com.yahoo.pulsar.broker.service.persistent.PersistentSubscription in project pulsar by yahoo.

the class PersistentTopicE2ETest method testConcurrentConsumerThreads.

// some race conditions needs to be handled
// disabling the test for now to not block commit jobs
@Test(enabled = false)
public void testConcurrentConsumerThreads() throws Exception {
    // test concurrent consumer threads on same consumerId
    final String topicName = "persistent://prop/use/ns-abc/topic3";
    final String subName = "sub3";
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setReceiverQueueSize(recvQueueSize);
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
                for (int i = 0; i < recvQueueSize / numConsumersThreads; i++) {
                    Message msg = consumer.receive();
                    consumer.acknowledge(msg);
                }
                return null;
            }
        });
    }
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < recvQueueSize * numConsumersThreads; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    barrier.await();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    // 1. cumulatively all threads drain the backlog
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 2. flow control works the same as single consumer single thread
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);
}