Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 1 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BookieClientStatsGenerator method generate.

private Map<String, Map<String, PendingBookieOpsStats>> generate() throws Exception {
    if (pulsar.getBrokerService() != null && pulsar.getBrokerService().getTopics() != null) {
        pulsar.getBrokerService().getTopics().forEach((name, topicFuture) -> {
            PersistentTopic persistentTopic = (PersistentTopic) topicFuture.getNow(null);
            if (persistentTopic != null) {
                DestinationName destinationName = DestinationName.get(persistentTopic.getName());
                put(destinationName, persistentTopic.getManagedLedger().getStats().getPendingBookieOpsStats());
            }
        });
    }
    return nsBookieClientStatsMap;
}

Example 2 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentQueueE2ETest method testCancelReadRequestOnLastDisconnect.

@Test(timeOut = 60000)
public void testCancelReadRequestOnLastDisconnect() throws Exception {
    String key = "testCancelReadRequestOnLastDisconnect";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(1000);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            consumer1.acknowledge(message1);
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
            consumer2.acknowledge(message2);
        }
        message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // 5. Close Consumer 1 and 2
    log.info("Consumer 1 closed");
    log.info("Consumer 2 closed");
    consumer1.close();
    consumer2.close();
    // 6. Producer produces more messages
    for (int i = totalMessages; i < 2 * totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 7. Consumer reconnects
    consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 8. Check number of messages received
    receivedConsumer1 = 0;
    message1 = consumer1.receive();
    while (message1 != null) {
        log.info("Consumer 1 Received: " + new String(message1.getData()));
        receivedConsumer1++;
        message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
    }
    log.info("Total receives by Consumer 2 = " + receivedConsumer2);
    assertEquals(receivedConsumer1, totalMessages);
}

Example 3 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentQueueE2ETest method testSharedSingleAckedNormalTopic.

@Test(timeOut = 300000)
public void testSharedSingleAckedNormalTopic() throws Exception {
    String key = "test1";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 50;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(10);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
        message1 = consumer1.receive(10000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(10000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // 5. Close Consumer 1
    log.info("Consumer 1 closed");
    consumer1.close();
    // 6. Consumer 1's unAcked messages should be sent to Consumer 2
    for (int i = 0; i < totalMessages; i++) {
        message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
        if (message2 == null) {
            log.info("Consumer 2 - No Message in Incoming Message Queue, will try again");
            continue;
        }
        log.info("Consumer 2 Received: " + new String(message2.getData()));
        receivedConsumer2 += 1;
    }
    log.info("Total receives by Consumer 2 = " + receivedConsumer2);
    assertEquals(receivedConsumer2, totalMessages);
}

Example 4 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method testConcurrentTopicDeleteAndUnsubscribe.

// @Test
public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                Thread.sleep(4, 700);
                log.info("deleter outcome is {}", topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                // Thread.sleep(2,0);
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                log.info("unsubscribe result : {}", topic.unsubscribe(successSubName).get());
                log.info("closing consumer..");
                ps.getConsumers().get(0).close();
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}

Example 5 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method testConcurrentTopicDeleteAndSubsUnsubscribe.

// @Test
public void testConcurrentTopicDeleteAndSubsUnsubscribe() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                Thread.sleep(4, 730);
                log.info("@@@@@@@@ DELETER TH");
                log.info("deleter outcome is " + topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                log.info("&&&&&&&&& UNSUBSCRIBER TH");
                // Thread.sleep(2,0);
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                log.info("unsubscribe result : " + ps.doUnsubscribe(ps.getConsumers().get(0)).get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}