Examples with ConsumerConfiguration com.yahoo.pulsar.client.api.ConsumerConfiguration used on opensource projects

Example 16 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class NamespaceServiceTest method testUnloadNamespaceBundleFailure.

@Test
public void testUnloadNamespaceBundleFailure() throws Exception {
    final String topicName = "persistent://my-property/use/my-ns/my-topic1";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    Consumer consumer = pulsarClient.subscribe(topicName, "my-subscriber-name", conf);
    ConcurrentOpenHashMap<String, CompletableFuture<Topic>> topics = pulsar.getBrokerService().getTopics();
    Topic spyTopic = spy(topics.get(topicName).get());
    topics.clear();
    CompletableFuture<Topic> topicFuture = CompletableFuture.completedFuture(spyTopic);
    // add mock topic
    topics.put(topicName, topicFuture);
    doAnswer(new Answer<CompletableFuture<Void>>() {

        @Override
        public CompletableFuture<Void> answer(InvocationOnMock invocation) throws Throwable {
            CompletableFuture<Void> result = new CompletableFuture<>();
            result.completeExceptionally(new RuntimeException("first time failed"));
            return result;
        }
    }).when(spyTopic).close();
    NamespaceBundle bundle = pulsar.getNamespaceService().getBundle(DestinationName.get(topicName));
    try {
        pulsar.getNamespaceService().unloadNamespaceBundle(bundle);
    } catch (Exception e) {
        // fail
        fail(e.getMessage());
    }
    try {
        pulsar.getLocalZkCache().getZooKeeper().getData(ServiceUnitZkUtils.path(bundle), null, null);
        fail("it should fail as node is not present");
    } catch (org.apache.zookeeper.KeeperException.NoNodeException e) {
    // ok
    }
}

Example 17 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class PersistentTopicE2ETest method testActiveSubscriptionWithCache.

/**
     * Validation: 1. validates active-cursor after active subscription 2. validate active-cursor with subscription 3.
     * unconsumed messages should be present into cache 4. cache and active-cursor should be empty once subscription is
     * closed
     *
     * @throws Exception
     */
@Test
public void testActiveSubscriptionWithCache() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int recvQueueSize = 4;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setReceiverQueueSize(recvQueueSize);
    // (1) Create subscription
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    // (2) Produce Messages
    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    // (3) Get Entry cache
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);
    /************* Validation on non-empty active-cursor **************/
    // (4) Get ActiveCursor : which is list of active subscription
    Iterable<ManagedCursor> activeCursors = ledger.getActiveCursors();
    ManagedCursor curosr = activeCursors.iterator().next();
    // (4.1) Validate: active Cursor must be non-empty
    assertNotNull(curosr);
    // (4.2) Validate: validate cursor name
    assertEquals(subName, curosr.getName());
    // (4.3) Validate: entryCache should have cached messages
    assertTrue(entryCache.getSize() != 0);
    /************* Validation on empty active-cursor **************/
    // (5) Close consumer: which (1)removes activeConsumer and (2)clears the entry-cache
    consumer.close();
    Thread.sleep(1000);
    // (5.1) Validate: active-consumer must be empty
    assertFalse(ledger.getActiveCursors().iterator().hasNext());
    // (5.2) Validate: Entry-cache must be cleared
    assertTrue(entryCache.getSize() == 0);
}

Example 18 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class PersistentTopicE2ETest method testConcurrentConsumerThreads.

// some race conditions needs to be handled
// disabling the test for now to not block commit jobs
@Test(enabled = false)
public void testConcurrentConsumerThreads() throws Exception {
    // test concurrent consumer threads on same consumerId
    final String topicName = "persistent://prop/use/ns-abc/topic3";
    final String subName = "sub3";
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setReceiverQueueSize(recvQueueSize);
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
                for (int i = 0; i < recvQueueSize / numConsumersThreads; i++) {
                    Message msg = consumer.receive();
                    consumer.acknowledge(msg);
                }
                return null;
            }
        });
    }
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < recvQueueSize * numConsumersThreads; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    barrier.await();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    // 1. cumulatively all threads drain the backlog
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 2. flow control works the same as single consumer single thread
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);
}

Example 19 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageRedelivery.

/**
     * Verify: Broker should not replay already acknowledged messages again and should clear them from messageReplay bucket
     *
     * 1. produce messages
     * 2. consume messages and ack all except 1 msg
     * 3. Verification: should replay only 1 unacked message
     */
@Test()
public void testMessageRedelivery() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int totalMessages = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    //(2) Consume and ack messages except first message
    Message unAckedMsg = null;
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        if (i == 0) {
            unAckedMsg = msg;
        } else {
            consumer.acknowledge(msg);
        }
    }
    consumer.redeliverUnacknowledgedMessages();
    // Verify: msg [L:0] must be redelivered
    try {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        assertEquals(new String(msg.getData()), new String(unAckedMsg.getData()));
    } catch (Exception e) {
        fail("msg should be redelivered ", e);
    }
    // Verify no other messages are redelivered
    msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);
    consumer.close();
    producer.close();
}

Example 20 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageReplay.

/**
     * Verify: 
     * 1. Broker should not replay already acknowledged messages 
     * 2. Dispatcher should not stuck while dispatching new messages due to previous-replay 
     * of invalid/already-acked messages
     * 
     * @throws Exception
     */
@Test
public void testMessageReplay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(1);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef.getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToReplay");
    replayMap.setAccessible(true);
    TreeSet<PositionImpl> messagesToReplay = Sets.newTreeSet();
    assertNotNull(subRef);
    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(new PositionImpl(msgId.getLedgerId(), msgId.getEntryId()));
        }
    }
    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);
    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);
    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(new PositionImpl(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId()));
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());
    consumer.close();
    producer.close();
}