Examples with PersistentDispatcherMultipleConsumers com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers used on opensource projects

Example 1 with PersistentDispatcherMultipleConsumers

use of com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers in project pulsar by yahoo.

the class PersistentTopicTest method testDispatcherMultiConsumerReadFailed.

@Test
public void testDispatcherMultiConsumerReadFailed() throws Exception {
    PersistentTopic topic = spy(new PersistentTopic(successTopicName, ledgerMock, brokerService));
    ManagedCursor cursor = mock(ManagedCursor.class);
    when(cursor.getName()).thenReturn("cursor");
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursor);
    dispatcher.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
    verify(topic, atLeast(1)).getBrokerService();
}

Example 2 with PersistentDispatcherMultipleConsumers

use of com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageReplay.

/**
     * Verify: 
     * 1. Broker should not replay already acknowledged messages 
     * 2. Dispatcher should not stuck while dispatching new messages due to previous-replay 
     * of invalid/already-acked messages
     * 
     * @throws Exception
     */
@Test
public void testMessageReplay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(1);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef.getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToReplay");
    replayMap.setAccessible(true);
    TreeSet<PositionImpl> messagesToReplay = Sets.newTreeSet();
    assertNotNull(subRef);
    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(new PositionImpl(msgId.getLedgerId(), msgId.getEntryId()));
        }
    }
    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);
    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);
    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(new PositionImpl(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId()));
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());
    consumer.close();
    producer.close();
}

Example 3 with PersistentDispatcherMultipleConsumers

use of com.yahoo.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers in project pulsar by yahoo.

the class PersistentDispatcherFailoverConsumerTest method testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel.

@Test
public void testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock);
    Consumer consumer1 = createConsumer(0, 2, 1);
    Consumer consumer2 = createConsumer(0, 2, 2);
    Consumer consumer3 = createConsumer(0, 2, 3);
    Consumer consumer4 = createConsumer(1, 2, 4);
    Consumer consumer5 = createConsumer(1, 1, 5);
    Consumer consumer6 = createConsumer(1, 2, 6);
    Consumer consumer7 = createConsumer(2, 1, 7);
    Consumer consumer8 = createConsumer(2, 1, 8);
    Consumer consumer9 = createConsumer(2, 1, 9);
    dispatcher.addConsumer(consumer1);
    dispatcher.addConsumer(consumer2);
    dispatcher.addConsumer(consumer3);
    dispatcher.addConsumer(consumer4);
    dispatcher.addConsumer(consumer5);
    dispatcher.addConsumer(consumer6);
    dispatcher.addConsumer(consumer7);
    dispatcher.addConsumer(consumer8);
    dispatcher.addConsumer(consumer9);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer5);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer8);
    // in between add upper priority consumer with more permits
    Consumer consumer10 = createConsumer(0, 2, 10);
    dispatcher.addConsumer(consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer9);
}