Examples with ConsumerConfiguration com.yahoo.pulsar.client.api.ConsumerConfiguration used on opensource projects

Example 21 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class PartitionedConsumerImpl method getInternalConsumerConfig.

private ConsumerConfiguration getInternalConsumerConfig() {
    ConsumerConfiguration internalConsumerConfig = new ConsumerConfiguration();
    internalConsumerConfig.setReceiverQueueSize(conf.getReceiverQueueSize());
    internalConsumerConfig.setSubscriptionType(conf.getSubscriptionType());
    internalConsumerConfig.setConsumerName(consumerName);
    if (conf.getAckTimeoutMillis() != 0) {
        internalConsumerConfig.setAckTimeout(conf.getAckTimeoutMillis(), TimeUnit.MILLISECONDS);
    }
    return internalConsumerConfig;
}

Example 22 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class AdminApiTest method testNamespaceBundleUnload.

@Test(dataProvider = "numBundles")
public void testNamespaceBundleUnload(Integer numBundles) throws Exception {
    admin.namespaces().createNamespace("prop-xyz/use/ns1-bundles", numBundles);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1-bundles"), Lists.newArrayList());
    // Force to create a destination
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1-bundles/ds2", 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1-bundles"), Lists.newArrayList("persistent://prop-xyz/use/ns1-bundles/ds2"));
    // create consumer and subscription
    ConsumerConfiguration conf = new ConsumerConfiguration();
    Consumer consumer = pulsarClient.subscribe("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"), Lists.newArrayList("my-sub"));
    // Create producer
    Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns1-bundles/ds2");
    for (int i = 0; i < 10; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    NamespaceBundle bundle = (NamespaceBundle) pulsar.getNamespaceService().getBundle(DestinationName.get("persistent://prop-xyz/use/ns1-bundles/ds2"));
    consumer.close();
    producer.close();
    admin.namespaces().unloadNamespaceBundle("prop-xyz/use/ns1-bundles", bundle.getBundleRange());
    // check that no one owns the namespace bundle
    assertFalse(pulsar.getNamespaceService().isServiceUnitOwned(bundle));
    assertFalse(otherPulsar.getNamespaceService().isServiceUnitOwned(bundle));
    LOG.info("--- RELOAD ---");
    // Force reload of namespace and wait for topic to be ready
    for (int i = 0; i < 30; i++) {
        try {
            admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1-bundles/ds2");
            break;
        } catch (PulsarAdminException e) {
            LOG.warn("Failed to get topic stats.. {}", e.getMessage());
            Thread.sleep(1000);
        }
    }
    admin.persistentTopics().deleteSubscription("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub");
    admin.persistentTopics().delete("persistent://prop-xyz/use/ns1-bundles/ds2");
}

Example 23 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class AdminApiTest method persistentTopicsCursorResetAfterReset.

@Test(dataProvider = "topicName")
public void persistentTopicsCursorResetAfterReset(String topicName) throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    topicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // create consumer and subscription
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 5, 0);
    // Allow at least 1ms for messages to have different timestamps
    Thread.sleep(1);
    long firstTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 3, 5);
    Thread.sleep(1);
    long secondTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 2, 8);
    List<Message> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    messages.forEach(message -> {
        LOG.info("Peeked message: {}", new String(message.getData()));
    });
    for (int i = 0; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
    }
    admin.persistentTopics().resetCursor(topicName, "my-sub", firstTimestamp);
    int receivedAfterReset = 0;
    // Should received messages from 4-9
    for (int i = 4; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(new String(message.getData()), expected);
    }
    assertEquals(receivedAfterReset, 6);
    // Reset at 2nd timestamp
    receivedAfterReset = 0;
    admin.persistentTopics().resetCursor(topicName, "my-sub", secondTimestamp);
    // Should received messages from 7-9
    for (int i = 7; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(new String(message.getData()), expected);
    }
    assertEquals(receivedAfterReset, 3);
    consumer.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList());
    admin.persistentTopics().delete(topicName);
}

Example 24 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class AdminApiTest method partitionedTopicsCursorReset.

@Test(dataProvider = "topicName")
public void partitionedTopicsCursorReset(String topicName) throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    topicName = "persistent://prop-xyz/use/ns1/" + topicName;
    admin.persistentTopics().createPartitionedTopic(topicName, 4);
    // create consumer and subscription
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
    List<String> destinations = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(destinations.size(), 4);
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 5, 0);
    Thread.sleep(1);
    long timestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 5, 5);
    for (int i = 0; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
    }
    // messages should still be available due to retention
    admin.persistentTopics().resetCursor(topicName, "my-sub", timestamp);
    Set<String> expectedMessages = Sets.newHashSet();
    Set<String> receivedMessages = Sets.newHashSet();
    for (int i = 4; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        expectedMessages.add("message-" + i);
        receivedMessages.add(new String(message.getData()));
    }
    receivedMessages.removeAll(expectedMessages);
    assertEquals(receivedMessages.size(), 0);
    consumer.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    admin.persistentTopics().deletePartitionedTopic(topicName);
}

Example 25 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class AdminApiTest method persistentTopics.

@Test(dataProvider = "topicName")
public void persistentTopics(String topicName) throws Exception {
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    final String persistentTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // Force to create a destination
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/" + topicName));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = client.subscribe(persistentTopicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 0);
    PersistentTopicInternalStats internalStats = admin.persistentTopics().getInternalStats(persistentTopicName);
    assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    List<Message> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
    assertEquals(messages.size(), 3);
    for (int i = 0; i < 3; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 15);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    admin.persistentTopics().skipMessages(persistentTopicName, "my-sub", 5);
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 5);
    admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    consumer.close();
    client.close();
    admin.persistentTopics().deleteSubscription(persistentTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList());
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet());
    assertEquals(topicStats.publishers.size(), 0);
    try {
        admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    } catch (NotFoundException e) {
    }
    admin.persistentTopics().delete(persistentTopicName);
    try {
        admin.persistentTopics().delete(persistentTopicName);
        fail("Should have received 404");
    } catch (NotFoundException e) {
    }
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
}