Examples with Consumer com.yahoo.pulsar.client.api.Consumer used on opensource projects

Example 31 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class AdminApiTest method persistentTopicsCursorResetAfterReset.

@Test(dataProvider = "topicName")
public void persistentTopicsCursorResetAfterReset(String topicName) throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    topicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // create consumer and subscription
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 5, 0);
    // Allow at least 1ms for messages to have different timestamps
    Thread.sleep(1);
    long firstTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 3, 5);
    Thread.sleep(1);
    long secondTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 2, 8);
    List<Message> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    messages.forEach(message -> {
        LOG.info("Peeked message: {}", new String(message.getData()));
    });
    for (int i = 0; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
    }
    admin.persistentTopics().resetCursor(topicName, "my-sub", firstTimestamp);
    int receivedAfterReset = 0;
    // Should received messages from 4-9
    for (int i = 4; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(new String(message.getData()), expected);
    }
    assertEquals(receivedAfterReset, 6);
    // Reset at 2nd timestamp
    receivedAfterReset = 0;
    admin.persistentTopics().resetCursor(topicName, "my-sub", secondTimestamp);
    // Should received messages from 7-9
    for (int i = 7; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(new String(message.getData()), expected);
    }
    assertEquals(receivedAfterReset, 3);
    consumer.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList());
    admin.persistentTopics().delete(topicName);
}

Example 32 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class AdminApiTest method partitionedTopicsCursorReset.

@Test(dataProvider = "topicName")
public void partitionedTopicsCursorReset(String topicName) throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    topicName = "persistent://prop-xyz/use/ns1/" + topicName;
    admin.persistentTopics().createPartitionedTopic(topicName, 4);
    // create consumer and subscription
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
    List<String> destinations = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(destinations.size(), 4);
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 5, 0);
    Thread.sleep(1);
    long timestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 5, 5);
    for (int i = 0; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
    }
    // messages should still be available due to retention
    admin.persistentTopics().resetCursor(topicName, "my-sub", timestamp);
    Set<String> expectedMessages = Sets.newHashSet();
    Set<String> receivedMessages = Sets.newHashSet();
    for (int i = 4; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        expectedMessages.add("message-" + i);
        receivedMessages.add(new String(message.getData()));
    }
    receivedMessages.removeAll(expectedMessages);
    assertEquals(receivedMessages.size(), 0);
    consumer.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    admin.persistentTopics().deletePartitionedTopic(topicName);
}

Example 33 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class AdminApiTest method persistentTopics.

@Test(dataProvider = "topicName")
public void persistentTopics(String topicName) throws Exception {
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    final String persistentTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // Force to create a destination
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/" + topicName));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = client.subscribe(persistentTopicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 0);
    PersistentTopicInternalStats internalStats = admin.persistentTopics().getInternalStats(persistentTopicName);
    assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    List<Message> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
    assertEquals(messages.size(), 3);
    for (int i = 0; i < 3; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 15);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    admin.persistentTopics().skipMessages(persistentTopicName, "my-sub", 5);
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 5);
    admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    consumer.close();
    client.close();
    admin.persistentTopics().deleteSubscription(persistentTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList());
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet());
    assertEquals(topicStats.publishers.size(), 0);
    try {
        admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    } catch (NotFoundException e) {
    }
    admin.persistentTopics().delete(persistentTopicName);
    try {
        admin.persistentTopics().delete(persistentTopicName);
        fail("Should have received 404");
    } catch (NotFoundException e) {
    }
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
}

Example 34 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ResendRequestTest method testSharedSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testSharedSingleAckedNormalTopic() throws Exception {
    String key = "testSharedSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(totalMessages / 2);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
        message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // 5. Send a resend request from Consumer 1
    log.info("Consumer 1 sent a resend request");
    consumer1.redeliverUnacknowledgedMessages();
    // 6. Consumer 1's unAcked messages should be sent to Consumer 1 or 2
    int receivedMessagesAfterRedelivery = 0;
    receivedConsumer1 = 0;
    message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            ++receivedMessagesAfterRedelivery;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
            ++receivedMessagesAfterRedelivery;
        }
        message1 = consumer1.receive(200, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(200, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Additional received = " + receivedMessagesAfterRedelivery);
    assertTrue(receivedMessagesAfterRedelivery > 0);
    assertEquals(receivedConsumer1 + receivedConsumer2, totalMessages);
}

Example 35 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ResendRequestTest method testFailoverInactiveConsumer.

@Test(timeOut = testTimeout)
public void testFailoverInactiveConsumer() throws Exception {
    String key = "testFailoverInactiveConsumer";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(10);
    conf.setSubscriptionType(SubscriptionType.Failover);
    conf.setConsumerName("consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    conf.setConsumerName("consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1;
    Message message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
    } while (message1 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 5. Consumer 2 asks for a redelivery but the request is ignored
    log.info("Consumer 2 asks for resend");
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    assertEquals(message1, null);
    assertEquals(message2, null);
}