Examples with Consumer com.yahoo.pulsar.client.api.Consumer used on opensource projects

Example 36 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ResendRequestTest method testExclusiveSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testExclusiveSingleAckedNormalTopic() throws Exception {
    String key = "testExclusiveSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    HashSet<MessageId> messageIdHashSet = new HashSet<MessageId>();
    HashSet<String> messageDataHashSet = new HashSet<String>();
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(7);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message = consumer.receive();
    log.info("Message received " + new String(message.getData()));
    for (int i = 1; i < totalMessages; i++) {
        Message msg = consumer.receive();
        log.info("Message received " + new String(msg.getData()));
        messageDataHashSet.add(new String(msg.getData()));
    }
    printIncomingMessageQueue(consumer);
    // 5. Ack 1st message and ask for resend
    consumer.acknowledge(message);
    log.info("Message acked " + new String(message.getData()));
    messageIdHashSet.add(message.getMessageId());
    messageDataHashSet.add(new String(message.getData()));
    consumer.redeliverUnacknowledgedMessages();
    log.info("Resend Messages Request sent");
    // 6. Check if messages resent in correct order
    for (int i = 0; i < totalMessages - 1; i++) {
        message = consumer.receive();
        log.info("Message received " + new String(message.getData()));
        if (i < 2) {
            messageIdHashSet.add(message.getMessageId());
            consumer.acknowledge(message);
        }
        log.info("Message acked " + new String(message.getData()));
        assertTrue(messageDataHashSet.contains(new String(message.getData())));
    }
    assertEquals(messageIdHashSet.size(), 3);
    assertEquals(messageDataHashSet.size(), totalMessages);
    printIncomingMessageQueue(consumer);
    // 7. Request resend 2nd time - you should receive 4 messages
    consumer.redeliverUnacknowledgedMessages();
    log.info("Resend Messages Request sent");
    message = consumer.receive(2000, TimeUnit.MILLISECONDS);
    while (message != null) {
        log.info("Message received " + new String(message.getData()));
        consumer.acknowledge(message);
        log.info("Message acked " + new String(message.getData()));
        messageIdHashSet.add(message.getMessageId());
        messageDataHashSet.add(new String(message.getData()));
        message = consumer.receive(5000, TimeUnit.MILLISECONDS);
    }
    assertEquals(messageIdHashSet.size(), totalMessages);
    assertEquals(messageDataHashSet.size(), totalMessages);
    printIncomingMessageQueue(consumer);
    // 9. Calling resend after acking all messages - expectin 0 messages
    consumer.redeliverUnacknowledgedMessages();
    assertEquals(consumer.receive(2000, TimeUnit.MILLISECONDS), null);
    // 10. Checking message contents
    for (int i = 0; i < totalMessages; i++) {
        assertTrue(messageDataHashSet.contains(messagePredicate + i));
    }
}

Example 37 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class BrokerClientIntegrationTest method testResetCursor.

@Test(timeOut = 10000, dataProvider = "subType")
public void testResetCursor(SubscriptionType subType) throws Exception {
    final RetentionPolicies policy = new RetentionPolicies(60, 52 * 1024);
    final DestinationName destName = DestinationName.get("persistent://my-property/use/my-ns/unacked-topic");
    final int warmup = 20;
    final int testSize = 150;
    final List<Message> received = new ArrayList<Message>();
    final ConsumerConfiguration consConfig = new ConsumerConfiguration();
    final String subsId = "sub";
    final NavigableMap<Long, TimestampEntryCount> publishTimeIdMap = new ConcurrentSkipListMap<>();
    consConfig.setSubscriptionType(subType);
    consConfig.setMessageListener((MessageListener) (Consumer consumer, Message msg) -> {
        try {
            synchronized (received) {
                received.add(msg);
            }
            consumer.acknowledge(msg);
            long publishTime = ((MessageImpl) msg).getPublishTime();
            log.info(" publish time is " + publishTime + "," + msg.getMessageId());
            TimestampEntryCount timestampEntryCount = publishTimeIdMap.computeIfAbsent(publishTime, (k) -> new TimestampEntryCount(publishTime));
            timestampEntryCount.incrementAndGet();
        } catch (final PulsarClientException e) {
            log.warn("Failed to ack!");
        }
    });
    admin.namespaces().setRetention(destName.getNamespace(), policy);
    Consumer consumer = pulsarClient.subscribe(destName.toString(), subsId, consConfig);
    final Producer producer = pulsarClient.createProducer(destName.toString());
    log.info("warm up started for " + destName.toString());
    // send warmup msgs
    byte[] msgBytes = new byte[1000];
    for (Integer i = 0; i < warmup; i++) {
        producer.send(msgBytes);
    }
    log.info("warm up finished.");
    // sleep to ensure receiving of msgs
    for (int n = 0; n < 10 && received.size() < warmup; n++) {
        Thread.sleep(100);
    }
    // validate received msgs
    Assert.assertEquals(received.size(), warmup);
    received.clear();
    // publish testSize num of msgs
    log.info("Sending more messages.");
    for (Integer n = 0; n < testSize; n++) {
        producer.send(msgBytes);
        Thread.sleep(1);
    }
    log.info("Sending more messages done.");
    Thread.sleep(3000);
    long begints = publishTimeIdMap.firstEntry().getKey();
    long endts = publishTimeIdMap.lastEntry().getKey();
    // find reset timestamp
    long timestamp = (endts - begints) / 2 + begints;
    timestamp = publishTimeIdMap.floorKey(timestamp);
    NavigableMap<Long, TimestampEntryCount> expectedMessages = new ConcurrentSkipListMap<>();
    expectedMessages.putAll(publishTimeIdMap.tailMap(timestamp, true));
    received.clear();
    log.info("reset cursor to " + timestamp + " for topic " + destName.toString() + " for subs " + subsId);
    log.info("issuing admin operation on " + admin.getServiceUrl().toString());
    List<String> subList = admin.persistentTopics().getSubscriptions(destName.toString());
    for (String subs : subList) {
        log.info("got sub " + subs);
    }
    publishTimeIdMap.clear();
    // reset the cursor to this timestamp
    Assert.assertTrue(subList.contains(subsId));
    admin.persistentTopics().resetCursor(destName.toString(), subsId, timestamp);
    consumer = pulsarClient.subscribe(destName.toString(), subsId, consConfig);
    Thread.sleep(3000);
    int totalExpected = 0;
    for (TimestampEntryCount tec : expectedMessages.values()) {
        totalExpected += tec.numMessages;
    }
    // validate that replay happens after the timestamp
    Assert.assertTrue(publishTimeIdMap.firstEntry().getKey() >= timestamp);
    consumer.close();
    producer.close();
    // validate that expected and received counts match
    int totalReceived = 0;
    for (TimestampEntryCount tec : publishTimeIdMap.values()) {
        totalReceived += tec.numMessages;
    }
    Assert.assertEquals(totalReceived, totalExpected, "did not receive all messages on replay after reset");
}

Example 38 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ConsumeBaseExceptionTest method testListener.

@Test
public void testListener() throws PulsarClientException {
    Consumer consumer = null;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setMessageListener((Consumer c, Message msg) -> {
    });
    consumer = pulsarClient.subscribe("persistent://prop/cluster/ns/topicName", "my-subscription", conf);
    Assert.assertTrue(consumer.receiveAsync().isCompletedExceptionally());
    try {
        consumer.receiveAsync().exceptionally(e -> {
            Assert.assertTrue(e instanceof PulsarClientException.InvalidConfigurationException);
            return null;
        }).get();
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 39 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class MessageIdTest method testChecksumReconnection.

@Test
public void testChecksumReconnection() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic1";
    // 1. producer connect
    ProducerImpl prod = (ProducerImpl) pulsarClient.createProducer(topicName);
    ProducerImpl producer = spy(prod);
    // mock: broker-doesn't support checksum (remote_version < brokerChecksumSupportedVersion) so, it forces
    // client-producer to perform checksum-strip from msg at reconnection
    doReturn(producer.brokerChecksumSupportedVersion() + 1).when(producer).brokerChecksumSupportedVersion();
    doAnswer(invocationOnMock -> prod.getState()).when(producer).getState();
    doAnswer(invocationOnMock -> prod.getClientCnx()).when(producer).getClientCnx();
    doAnswer(invocationOnMock -> prod.cnx()).when(producer).cnx();
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    stopBroker();
    // stop timer to auto-reconnect as let spy-Producer connect to broker
    // manually so, spy-producer object can get
    // mock-value from brokerChecksumSupportedVersion
    ((PulsarClientImpl) pulsarClient).timer().stop();
    // set clientCnx mock to get non-checksum supported version
    ClientCnx mockClientCnx = spy(new ClientCnx((PulsarClientImpl) pulsarClient));
    doReturn(producer.brokerChecksumSupportedVersion() - 1).when(mockClientCnx).getRemoteEndpointProtocolVersion();
    prod.setClientCnx(mockClientCnx);
    Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
    Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
    CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
    // corrupt the message
    // new content would be
    msg2.getData()[msg2.getData().length - 1] = '3';
    // 'message-3'
    // unset mock
    prod.setClientCnx(null);
    // Restart the broker to have the messages published
    startBroker();
    // grab broker connection with mocked producer which has higher version
    // compare to broker
    prod.grabCnx();
    try {
        // it should not fail: as due to unsupported version of broker:
        // client removes checksum and broker should
        // ignore the checksum validation
        future1.get(10, TimeUnit.SECONDS);
        future2.get(10, TimeUnit.SECONDS);
    } catch (Exception e) {
        e.printStackTrace();
        fail("Broker shouldn't verify checksum for corrupted message and it shouldn't fail");
    }
    ((ConsumerImpl) consumer).grabCnx();
    // We should only receive msg1
    Message msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-1");
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-3");
}

Example 40 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class MessageIdTest method producerSendAsync.

@Test(timeOut = 10000)
public void producerSendAsync() throws PulsarClientException {
    // 1. Basic Config
    String key = "producerSendAsync";
    final String topicName = "persistent://prop/cluster/namespace/topic-" + key;
    final String subscriptionName = "my-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int numberOfMessages = 30;
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    // 4. Publish message and get message id
    Set<MessageId> messageIds = new HashSet();
    List<Future<MessageId>> futures = new ArrayList();
    for (int i = 0; i < numberOfMessages; i++) {
        String message = messagePredicate + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    MessageIdImpl previousMessageId = null;
    for (Future<MessageId> f : futures) {
        try {
            MessageIdImpl currentMessageId = (MessageIdImpl) f.get();
            if (previousMessageId != null) {
                Assert.assertTrue(currentMessageId.compareTo(previousMessageId) > 0, "Message Ids should be in ascending order");
            }
            messageIds.add(currentMessageId);
            previousMessageId = currentMessageId;
        } catch (Exception e) {
            Assert.fail("Failed to publish message, Exception: " + e.getMessage());
        }
    }
    // 4. Check if message Ids are correct
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
    for (int i = 0; i < numberOfMessages; i++) {
        Message message = consumer.receive();
        Assert.assertEquals(new String(message.getData()), messagePredicate + i);
        MessageId messageId = message.getMessageId();
        Assert.assertTrue(messageIds.remove(messageId), "Failed to receive message");
    }
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), 0, "Not all messages received successfully");
    consumer.unsubscribe();
}