Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 46 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testFailoverSubscription.

@Test
public void testFailoverSubscription() throws Exception {
    PersistentTopic topic1 = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    CommandSubscribe cmd1 = CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Failover).build();
    // 1. Subscribe with non partition topic
    Future<Consumer> f1 = topic1.subscribe(serverCnx, cmd1.getSubscription(), cmd1.getConsumerId(), cmd1.getSubType(), 0, cmd1.getConsumerName());
    f1.get();
    // 2. Subscribe with partition topic
    PersistentTopic topic2 = new PersistentTopic(successPartitionTopicName, ledgerMock, brokerService);
    CommandSubscribe cmd2 = CommandSubscribe.newBuilder().setConsumerId(1).setConsumerName("C1").setTopic(successPartitionTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Failover).build();
    Future<Consumer> f2 = topic2.subscribe(serverCnx, cmd2.getSubscription(), cmd2.getConsumerId(), cmd2.getSubType(), 0, cmd2.getConsumerName());
    f2.get();
    // 3. Subscribe and create second consumer
    CommandSubscribe cmd3 = CommandSubscribe.newBuilder().setConsumerId(2).setConsumerName("C2").setTopic(successPartitionTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Failover).build();
    Future<Consumer> f3 = topic2.subscribe(serverCnx, cmd3.getSubscription(), cmd3.getConsumerId(), cmd3.getSubType(), 0, cmd3.getConsumerName());
    f3.get();
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerId(), 1);
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerName(), "C1");
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(1).consumerId(), 2);
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(1).consumerName(), "C2");
    // 4. Subscribe and create third duplicate consumer
    CommandSubscribe cmd4 = CommandSubscribe.newBuilder().setConsumerId(3).setConsumerName("C1").setTopic(successPartitionTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Failover).build();
    Future<Consumer> f4 = topic2.subscribe(serverCnx, cmd4.getSubscription(), cmd4.getConsumerId(), cmd4.getSubType(), 0, cmd4.getConsumerName());
    f4.get();
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerId(), 1);
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerName(), "C1");
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(1).consumerId(), 3);
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(1).consumerName(), "C1");
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(2).consumerId(), 2);
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(2).consumerName(), "C2");
    // 5. Subscribe on partition topic with existing consumer id and different sub type
    CommandSubscribe cmd5 = CommandSubscribe.newBuilder().setConsumerId(2).setConsumerName("C1").setTopic(successPartitionTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
    Future<Consumer> f5 = topic2.subscribe(serverCnx, cmd5.getSubscription(), cmd5.getConsumerId(), cmd5.getSubType(), 0, cmd5.getConsumerName());
    try {
        f5.get();
        fail("should fail with exception");
    } catch (ExecutionException ee) {
        // Expected
        assertTrue(ee.getCause() instanceof BrokerServiceException.SubscriptionBusyException);
    }
    // 6. Subscribe on partition topic with different sub name, type and different consumer id
    CommandSubscribe cmd6 = CommandSubscribe.newBuilder().setConsumerId(4).setConsumerName("C3").setTopic(successPartitionTopicName).setSubscription(successSubName2).setRequestId(1).setSubType(SubType.Exclusive).build();
    Future<Consumer> f6 = topic2.subscribe(serverCnx, cmd6.getSubscription(), cmd6.getConsumerId(), cmd6.getSubType(), 0, cmd6.getConsumerName());
    f6.get();
    // 7. unsubscribe exclusive sub
    Future<Void> f7 = topic2.unsubscribe(successSubName2);
    f7.get();
    assertNull(topic2.getPersistentSubscription(successSubName2));
    // 8. unsubscribe active consumer from shared sub.
    PersistentSubscription sub = topic2.getPersistentSubscription(successSubName);
    Consumer cons = sub.getDispatcher().getConsumers().get(0);
    sub.removeConsumer(cons);
    // Verify second consumer become active
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerId(), 3);
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerName(), "C1");
    // 9. unsubscribe active consumer from shared sub.
    cons = sub.getDispatcher().getConsumers().get(0);
    sub.removeConsumer(cons);
    // Verify second consumer become active
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerId(), 2);
    assertEquals(topic2.getPersistentSubscription(successSubName).getDispatcher().getConsumers().get(0).consumerName(), "C2");
    // 10. unsubscribe shared sub
    Future<Void> f8 = topic2.unsubscribe(successSubName);
    f8.get();
    assertNull(topic2.getPersistentSubscription(successSubName));
}

Example 47 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testConcurrentTopicAndSubscriptionDelete.

// @Test
public void testConcurrentTopicAndSubscriptionDelete() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    CommandSubscribe cmd = CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        @Override
        public void run() {
            try {
                barrier.await();
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                Thread.sleep(5, 0);
                log.info("deleter outcome is {}", topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        @Override
        public void run() {
            try {
                barrier.await();
                // do subscription delete
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                // Thread.sleep(5,0);
                log.info("unsubscriber outcome is {}", ps.doUnsubscribe(ps.getConsumers().get(0)).get());
            // assertFalse(ps.delete().isCompletedExceptionally());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}

Example 48 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testUbsubscribeRaceConditions.

@Test
public void testUbsubscribeRaceConditions() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, cursorMock);
    Consumer consumer1 = new Consumer(sub, SubType.Exclusive, 1, /* consumer id */
    0, "Cons1", /* consumer name */
    50000, serverCnx, "myrole-1");
    sub.addConsumer(consumer1);
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteCursorCallback) invocationOnMock.getArguments()[1]).deleteCursorComplete(null);
            Thread.sleep(1000);
            return null;
        }
    }).when(ledgerMock).asyncDeleteCursor(matches(".*success.*"), any(DeleteCursorCallback.class), anyObject());
    ExecutorService executor = Executors.newCachedThreadPool();
    executor.submit(() -> {
        sub.doUnsubscribe(consumer1);
        return null;
    }).get();
    try {
        Thread.sleep(10);
        /* delay to ensure that the ubsubscribe gets executed first */
        Consumer consumer2 = new Consumer(sub, SubType.Exclusive, 2, /* consumer id */
        0, "Cons2", /* consumer name */
        50000, serverCnx, "myrole-1");
    } catch (BrokerServiceException e) {
        assertTrue(e instanceof BrokerServiceException.SubscriptionFencedException);
    }
}

Example 49 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testSubscribeUnsubscribe.

@Test
public void testSubscribeUnsubscribe() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    CommandSubscribe cmd = CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
    // 1. simple subscribe
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    // 2. duplicate subscribe
    Future<Consumer> f2 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    try {
        f2.get();
        fail("should fail with exception");
    } catch (ExecutionException ee) {
        // Expected
        assertTrue(ee.getCause() instanceof BrokerServiceException.ConsumerBusyException);
    }
    // 3. simple unsubscribe
    Future<Void> f3 = topic.unsubscribe(successSubName);
    f3.get();
    assertNull(topic.getPersistentSubscription(successSubName));
}

Example 50 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerWithFixedBatchSizeAndTime.

@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType compressionType) throws Exception {
    int numMsgs = 100;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchSizeAndTime";
    final String subscriptionName = "time-size-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(5);
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    Random random = new Random();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        // put a random sleep from 0 to 3 ms
        Thread.sleep(random.nextInt(4));
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    LOG.info("Sent {} messages, backlog is {} messages", numMsgs, topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog());
    assertTrue(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog() < numMsgs);
    producer.close();
}