Examples with ConsumerConfiguration com.yahoo.pulsar.client.api.ConsumerConfiguration used on opensource projects

Example 56 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class AdminApiTest method persistentTopicsInvalidCursorReset.

@Test
public void persistentTopicsInvalidCursorReset() throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    String topicName = "persistent://prop-xyz/use/ns1/invalidcursorreset";
    // Force to create a destination
    publishMessagesOnPersistentTopic(topicName, 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList(topicName));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = client.subscribe(topicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 10);
    List<Message> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
    }
    // use invalid timestamp
    try {
        admin.persistentTopics().resetCursor(topicName, "my-sub", System.currentTimeMillis() - 190000);
    } catch (Exception e) {
        // fail the test
        throw e;
    }
    admin.persistentTopics().resetCursor(topicName, "my-sub", System.currentTimeMillis() + 90000);
    consumer = client.subscribe(topicName, "my-sub", conf);
    consumer.close();
    client.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList());
    admin.persistentTopics().delete(topicName);
}

Example 57 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class AdminApiTest method partitionedTopics.

@Test(dataProvider = "topicName")
public void partitionedTopics(String topicName) throws Exception {
    final String partitionedTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 4);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 4);
    // check if the virtual topic doesn't get created
    List<String> destinations = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(destinations.size(), 0);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata("persistent://prop-xyz/use/ns1/ds2").partitions, 0);
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = client.subscribe(partitionedTopicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
    Consumer consumer1 = client.subscribe(partitionedTopicName, "my-sub-1", conf);
    assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(partitionedTopicName)), Sets.newHashSet("my-sub", "my-sub-1"));
    consumer1.close();
    admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub-1");
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
    ProducerConfiguration prodConf = new ProducerConfiguration();
    prodConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = client.createProducer(partitionedTopicName, prodConf);
    for (int i = 0; i < 10; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    assertEquals(Sets.newHashSet(admin.persistentTopics().getList("prop-xyz/use/ns1")), Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1", partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
    // test cumulative stats for partitioned topic
    PartitionedTopicStats topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, false);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 1);
    assertEquals(topicStats.partitions, Maps.newHashMap());
    // test per partition stats for partitioned topic
    topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, true);
    assertEquals(topicStats.metadata.partitions, 4);
    assertEquals(topicStats.partitions.keySet(), Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1", partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
    PersistentTopicStats partitionStats = topicStats.partitions.get(partitionedTopicName + "-partition-0");
    assertEquals(partitionStats.publishers.size(), 1);
    assertEquals(partitionStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(partitionStats.subscriptions.get("my-sub").msgBacklog, 3, 1);
    try {
        admin.persistentTopics().skipMessages(partitionedTopicName, "my-sub", 5);
        fail("skip messages for partitioned topics should fail");
    } catch (Exception e) {
    // ok
    }
    admin.persistentTopics().skipAllMessages(partitionedTopicName, "my-sub");
    topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, false);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    producer.close();
    consumer.close();
    admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList());
    try {
        admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 32);
        fail("Should have failed as the partitioned topic already exists");
    } catch (ConflictException ce) {
    }
    producer = client.createProducer(partitionedTopicName);
    destinations = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(destinations.size(), 4);
    try {
        admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
        fail("The topic is busy");
    } catch (PreconditionFailedException pfe) {
    // ok
    }
    producer.close();
    client.close();
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 0);
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 32);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 32);
    try {
        admin.persistentTopics().deletePartitionedTopic("persistent://prop-xyz/use/ns1/ds2");
        fail("Should have failed as the partitioned topic was not created");
    } catch (NotFoundException nfe) {
    }
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
    // delete a partitioned topic in a global namespace
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 4);
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
}

Example 58 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class AdminApiTest method testPersistentTopicsExpireMessages.

/**
     * Verify: PersistentTopics.expireMessages()/expireMessagesForAllSubscriptions()
     * 1. Created multiple shared subscriptions and publisher on topic
     * 2. Publish messages on the topic
     * 3. expire message on sub-1 : backlog for sub-1 must be 0
     * 4. expire message on all subscriptions: backlog for all subscription must be 0
     *
     * @throws Exception
     */
@Test
public void testPersistentTopicsExpireMessages() throws Exception {
    // Force to create a destination
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/ds2"));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub1", conf);
    Consumer consumer2 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub2", conf);
    Consumer consumer3 = client.subscribe("persistent://prop-xyz/use/ns1/ds2", "my-sub3", conf);
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1/ds2").size(), 3);
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
    // wait for 1 seconds to expire message
    Thread.sleep(1000);
    admin.persistentTopics().expireMessages("persistent://prop-xyz/use/ns1/ds2", "my-sub1", 1);
    // wait for 1 seconds to execute expire message as it is async
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
    admin.persistentTopics().expireMessagesForAllSubscriptions("persistent://prop-xyz/use/ns1/ds2", 1);
    // wait for 1 seconds to execute expire message as it is async
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 0);
    consumer1.close();
    consumer2.close();
    consumer3.close();
}

Example 59 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class BrokerServiceTest method testBrokerServicePersistentTopicStats.

@Test
public void testBrokerServicePersistentTopicStats() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/successTopic";
    final String subName = "successSub";
    PersistentTopicStats stats;
    PersistentSubscriptionStats subStats;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);
    Producer producer = pulsarClient.createProducer(topicName);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);
    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);
    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    Message msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    assertEquals(subStats.msgBacklog, 0);
}

Example 60 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class BrokerServiceTest method testTlsEnabled.

@Test
public void testTlsEnabled() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/newTopic";
    final String subName = "newSub";
    ClientConfiguration clientConfig;
    ConsumerConfiguration consumerConfig;
    Consumer consumer;
    conf.setAuthenticationEnabled(false);
    conf.setTlsEnabled(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    restartBroker();
    // Case 1: Access without TLS
    PulsarClient pulsarClient = null;
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrl.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
    // Case 2: Access with TLS (Allow insecure TLS connection)
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(true);
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
    // Case 3: Access with TLS (Disallow insecure TLS connection)
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(false);
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
        fail("should fail");
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("General SSLEngine problem"));
    } finally {
        pulsarClient.close();
    }
    // Case 4: Access with TLS (Use trusted certificates)
    try {
        clientConfig = new ClientConfiguration();
        clientConfig.setUseTls(true);
        clientConfig.setTlsAllowInsecureConnection(false);
        clientConfig.setTlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH);
        clientConfig.setStatsInterval(0, TimeUnit.SECONDS);
        pulsarClient = PulsarClient.create(brokerUrlTls.toString(), clientConfig);
        consumerConfig = new ConsumerConfiguration();
        consumerConfig.setSubscriptionType(SubscriptionType.Exclusive);
        consumer = pulsarClient.subscribe(topicName, subName, consumerConfig);
        consumer.close();
    } catch (Exception e) {
        fail("should not fail");
    } finally {
        pulsarClient.close();
    }
}