Examples with MessageIdImpl com.yahoo.pulsar.client.impl.MessageIdImpl used on opensource projects

Example 1 with MessageIdImpl

use of com.yahoo.pulsar.client.impl.MessageIdImpl in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageReplay.

/**
     * Verify: 
     * 1. Broker should not replay already acknowledged messages 
     * 2. Dispatcher should not stuck while dispatching new messages due to previous-replay 
     * of invalid/already-acked messages
     * 
     * @throws Exception
     */
@Test
public void testMessageReplay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(1);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef.getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToReplay");
    replayMap.setAccessible(true);
    TreeSet<PositionImpl> messagesToReplay = Sets.newTreeSet();
    assertNotNull(subRef);
    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(new PositionImpl(msgId.getLedgerId(), msgId.getEntryId()));
        }
    }
    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);
    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);
    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(new PositionImpl(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId()));
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());
    consumer.close();
    producer.close();
}

Example 2 with MessageIdImpl

use of com.yahoo.pulsar.client.impl.MessageIdImpl in project pulsar by yahoo.

the class MessageIdSerialization method testProtobufSerialization1.

@Test
void testProtobufSerialization1() throws Exception {
    MessageId id = new MessageIdImpl(1, 2, 3);
    byte[] serializedId = id.toByteArray();
    assertEquals(MessageId.fromByteArray(serializedId), id);
}

Example 3 with MessageIdImpl

use of com.yahoo.pulsar.client.impl.MessageIdImpl in project pulsar by yahoo.

the class MessageIdSerialization method testProtobufSerialization2.

@Test
void testProtobufSerialization2() throws Exception {
    MessageId id = new MessageIdImpl(1, 2, -1);
    byte[] serializedId = id.toByteArray();
    assertEquals(MessageId.fromByteArray(serializedId), id);
}

Example 4 with MessageIdImpl

use of com.yahoo.pulsar.client.impl.MessageIdImpl in project pulsar by yahoo.

the class PersistentTopicE2ETest method testProducerReturnedMessageId.

@Test
public void testProducerReturnedMessageId() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-xyz";
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    long ledgerId = managedLedger.getLedgersInfoAsList().get(0).getLedgerId();
    // 2. producer publish messages
    final int SyncMessages = 10;
    for (int i = 0; i < SyncMessages; i++) {
        String message = "my-message-" + i;
        MessageId receivedMessageId = producer.send(message.getBytes());
        assertEquals(receivedMessageId, new MessageIdImpl(ledgerId, i, -1));
    }
    // 3. producer publish messages async
    final int AsyncMessages = 10;
    final CountDownLatch counter = new CountDownLatch(AsyncMessages);
    for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
        String content = "my-message-" + i;
        Message msg = MessageBuilder.create().setContent(content.getBytes()).build();
        final int index = i;
        producer.sendAsync(msg).thenRun(() -> {
            assertEquals(msg.getMessageId(), new MessageIdImpl(ledgerId, index, -1));
            counter.countDown();
        }).exceptionally((ex) -> {
            return null;
        });
    }
    counter.await();
    // 4. producer disconnect
    producer.close();
}

Example 5 with MessageIdImpl

use of com.yahoo.pulsar.client.impl.MessageIdImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause.

/**
     * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
     * blocked due to unacked messsages
     *
     * Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking
     *
     * @throws Exception
     */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 20;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), unAckedMessagesBufferSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}