Search in sources :

Example 1 with ConsumerBuilder

use of org.apache.pulsar.client.api.ConsumerBuilder in project incubator-pulsar by apache.

the class PerformanceConsumer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-consumer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
        if (arguments.useTls == false) {
            arguments.useTls = Boolean.parseBoolean(prop.getProperty("useTls"));
        }
        if (isBlank(arguments.tlsTrustCertsFilePath)) {
            arguments.tlsTrustCertsFilePath = prop.getProperty("tlsTrustCertsFilePath", "");
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments));
    final TopicName prefixTopicName = TopicName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    MessageListener<byte[]> listener = (consumer, msg) -> {
        messagesReceived.increment();
        bytesReceived.add(msg.getData().length);
        if (limiter != null) {
            limiter.acquire();
        }
        long latencyMillis = System.currentTimeMillis() - msg.getPublishTime();
        if (latencyMillis >= 0) {
            recorder.recordValue(latencyMillis);
            cumulativeRecorder.recordValue(latencyMillis);
        }
        consumer.acknowledgeAsync(msg);
    };
    ClientBuilder clientBuilder = // 
    PulsarClient.builder().serviceUrl(// 
    arguments.serviceURL).connectionsPerBroker(// 
    arguments.maxConnections).statsInterval(arguments.statsIntervalSeconds, // 
    TimeUnit.SECONDS).ioThreads(// 
    Runtime.getRuntime().availableProcessors()).enableTls(// 
    arguments.useTls).tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientBuilder.authentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = clientBuilder.build();
    class EncKeyReader implements CryptoKeyReader {

        EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();

        EncKeyReader(byte[] value) {
            keyInfo.setKey(value);
        }

        @Override
        public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
            return null;
        }

        @Override
        public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
            if (keyName.equals(arguments.encKeyName)) {
                return keyInfo;
            }
            return null;
        }
    }
    List<Future<Consumer<byte[]>>> futures = Lists.newArrayList();
    ConsumerBuilder<byte[]> consumerBuilder = // 
    pulsarClient.newConsumer().messageListener(// 
    listener).receiverQueueSize(// 
    arguments.receiverQueueSize).subscriptionType(arguments.subscriptionType);
    if (arguments.encKeyName != null) {
        byte[] pKey = Files.readAllBytes(Paths.get(arguments.encKeyFile));
        EncKeyReader keyReader = new EncKeyReader(pKey);
        consumerBuilder.cryptoKeyReader(keyReader);
    }
    for (int i = 0; i < arguments.numTopics; i++) {
        final TopicName topicName = (arguments.numTopics == 1) ? prefixTopicName : TopicName.get(String.format("%s-%d", prefixTopicName, i));
        log.info("Adding {} consumers on topic {}", arguments.numConsumers, topicName);
        for (int j = 0; j < arguments.numConsumers; j++) {
            String subscriberName;
            if (arguments.numConsumers > 1) {
                subscriberName = String.format("%s-%d", arguments.subscriberName, j);
            } else {
                subscriberName = arguments.subscriberName;
            }
            futures.add(consumerBuilder.clone().topic(topicName.toString()).subscriptionName(subscriberName).subscribeAsync());
        }
    }
    for (Future<Consumer<byte[]>> future : futures) {
        future.get();
    }
    log.info("Start receiving from {} consumers on {} topics", arguments.numConsumers, arguments.numTopics);
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput received: {}  msg/s -- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", dec.format(rate), dec.format(throughput), dec.format(reportHistogram.getMean()), (long) reportHistogram.getValueAtPercentile(50), (long) reportHistogram.getValueAtPercentile(95), (long) reportHistogram.getValueAtPercentile(99), (long) reportHistogram.getValueAtPercentile(99.9), (long) reportHistogram.getValueAtPercentile(99.99), (long) reportHistogram.getMaxValue());
        reportHistogram.reset();
        oldTime = now;
    }
    pulsarClient.close();
}
Also used : LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) ParameterException(com.beust.jcommander.ParameterException) Parameter(com.beust.jcommander.Parameter) LoggerFactory(org.slf4j.LoggerFactory) ConsumerBuilder(org.apache.pulsar.client.api.ConsumerBuilder) RateLimiter(com.google.common.util.concurrent.RateLimiter) Future(java.util.concurrent.Future) Lists(com.google.common.collect.Lists) Map(java.util.Map) Recorder(org.HdrHistogram.Recorder) EncryptionKeyInfo(org.apache.pulsar.client.api.EncryptionKeyInfo) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Properties(java.util.Properties) Logger(org.slf4j.Logger) Files(java.nio.file.Files) ObjectWriter(com.fasterxml.jackson.databind.ObjectWriter) MessageListener(org.apache.pulsar.client.api.MessageListener) DecimalFormat(java.text.DecimalFormat) JCommander(com.beust.jcommander.JCommander) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) FileInputStream(java.io.FileInputStream) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) TimeUnit(java.util.concurrent.TimeUnit) Histogram(org.HdrHistogram.Histogram) Consumer(org.apache.pulsar.client.api.Consumer) CryptoKeyReader(org.apache.pulsar.client.api.CryptoKeyReader) List(java.util.List) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Paths(java.nio.file.Paths) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Histogram(org.HdrHistogram.Histogram) EncryptionKeyInfo(org.apache.pulsar.client.api.EncryptionKeyInfo) Properties(java.util.Properties) CryptoKeyReader(org.apache.pulsar.client.api.CryptoKeyReader) Consumer(org.apache.pulsar.client.api.Consumer) JCommander(com.beust.jcommander.JCommander) ParameterException(com.beust.jcommander.ParameterException) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) ObjectWriter(com.fasterxml.jackson.databind.ObjectWriter) FileInputStream(java.io.FileInputStream) RateLimiter(com.google.common.util.concurrent.RateLimiter) TopicName(org.apache.pulsar.common.naming.TopicName) Future(java.util.concurrent.Future) Map(java.util.Map)

Example 2 with ConsumerBuilder

use of org.apache.pulsar.client.api.ConsumerBuilder in project incubator-pulsar by apache.

the class PulsarKafkaConsumer method subscribe.

@Override
public void subscribe(Collection<String> topics, ConsumerRebalanceListener callback) {
    List<CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>>> futures = new ArrayList<>();
    List<TopicPartition> topicPartitions = new ArrayList<>();
    try {
        for (String topic : topics) {
            // Create individual subscription on each partition, that way we can keep using the
            // acknowledgeCumulative()
            int numberOfPartitions = ((PulsarClientImpl) client).getNumberOfPartitions(topic).get();
            ConsumerBuilder<byte[]> consumerBuilder = PulsarConsumerKafkaConfig.getConsumerBuilder(client, properties);
            consumerBuilder.subscriptionType(SubscriptionType.Failover);
            consumerBuilder.messageListener(this);
            consumerBuilder.subscriptionName(groupId);
            if (numberOfPartitions > 1) {
                // Subscribe to each partition
                consumerBuilder.consumerName(ConsumerName.generateRandomName());
                for (int i = 0; i < numberOfPartitions; i++) {
                    String partitionName = TopicName.get(topic).getPartition(i).toString();
                    CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>> future = consumerBuilder.clone().topic(partitionName).subscribeAsync();
                    int partitionIndex = i;
                    TopicPartition tp = new TopicPartition(topic, partitionIndex);
                    future.thenAccept(consumer -> consumers.putIfAbsent(tp, consumer));
                    futures.add(future);
                    topicPartitions.add(tp);
                }
            } else {
                // Topic has a single partition
                CompletableFuture<org.apache.pulsar.client.api.Consumer<byte[]>> future = consumerBuilder.topic(topic).subscribeAsync();
                TopicPartition tp = new TopicPartition(topic, 0);
                future.thenAccept(consumer -> consumers.putIfAbsent(tp, consumer));
                futures.add(future);
                topicPartitions.add(tp);
            }
        }
        // Wait for all consumers to be ready
        futures.forEach(CompletableFuture::join);
        // Notify the listener is now owning all topics/partitions
        if (callback != null) {
            callback.onPartitionsAssigned(topicPartitions);
        }
    } catch (Exception e) {
        // Close all consumer that might have been successfully created
        futures.forEach(f -> {
            try {
                f.get().close();
            } catch (Exception e1) {
            // Ignore. Consumer already had failed
            }
        });
        throw new RuntimeException(e);
    }
}
Also used : TopicName(org.apache.pulsar.common.naming.TopicName) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) TimeoutException(java.util.concurrent.TimeoutException) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) ConsumerBuilder(org.apache.pulsar.client.api.ConsumerBuilder) PulsarConsumerKafkaConfig(org.apache.pulsar.client.kafka.compat.PulsarConsumerKafkaConfig) Message(org.apache.pulsar.client.api.Message) PulsarClientKafkaConfig(org.apache.pulsar.client.kafka.compat.PulsarClientKafkaConfig) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) ConsumerName(org.apache.pulsar.client.util.ConsumerName) StringDeserializer(org.apache.kafka.common.serialization.StringDeserializer) Map(java.util.Map) Metric(org.apache.kafka.common.Metric) MetricName(org.apache.kafka.common.MetricName) Deserializer(org.apache.kafka.common.serialization.Deserializer) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ProducerConfig(org.apache.kafka.clients.producer.ProducerConfig) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) TimestampType(org.apache.kafka.common.record.TimestampType) TopicPartition(org.apache.kafka.common.TopicPartition) Properties(java.util.Properties) MessageIdUtils(org.apache.pulsar.client.kafka.compat.MessageIdUtils) Collection(java.util.Collection) MessageListener(org.apache.pulsar.client.api.MessageListener) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) BlockingQueue(java.util.concurrent.BlockingQueue) PartitionInfo(org.apache.kafka.common.PartitionInfo) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) Collectors(java.util.stream.Collectors) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) Base64(java.util.Base64) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MessageId(org.apache.pulsar.client.api.MessageId) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Pattern(java.util.regex.Pattern) ArrayList(java.util.ArrayList) TimeoutException(java.util.concurrent.TimeoutException) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ExecutionException(java.util.concurrent.ExecutionException) CompletableFuture(java.util.concurrent.CompletableFuture) TopicPartition(org.apache.kafka.common.TopicPartition)

Example 3 with ConsumerBuilder

use of org.apache.pulsar.client.api.ConsumerBuilder in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testRoundRobinBatchDistribution.

// this test is good to have to see the distribution, but every now and then it gets slightly different than the
// expected numbers. keeping this disabled to not break the build, but nevertheless this gives good insight into
// how the round robin distribution algorithm is behaving
@Test(enabled = false)
public void testRoundRobinBatchDistribution() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic5";
    final String subName = "sub5";
    final int numMsgs = 137;
    /* some random number different than default batch size of 100 */
    final AtomicInteger counter1 = new AtomicInteger(0);
    final AtomicInteger counter2 = new AtomicInteger(0);
    final AtomicInteger counter3 = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(numMsgs * 3);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter1.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter2.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    Consumer<byte[]> consumer3 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter1.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs * 3; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    latch.await(1, TimeUnit.SECONDS);
    /*
         * total messages = 137 * 3 = 411 Each consumer has 10 permits. There will be 411 / 3*10 = 13 full distributions
         * i.e. each consumer will get 130 messages. In the 14th round, the balance is 411 - 130*3 = 21. Two consumers
         * will get another batch of 10 messages (Total: 140) and the 3rd one will get the last one (Total: 131)
         */
    assertTrue(CollectionUtils.subtract(Lists.newArrayList(140, 140, 131), Lists.newArrayList(counter1.get(), counter2.get(), counter3.get())).isEmpty());
    consumer1.close();
    consumer2.close();
    consumer3.close();
    admin.persistentTopics().delete(topicName);
}
Also used : SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) Assert.assertNull(org.testng.Assert.assertNull) PersistentTopicStats(org.apache.pulsar.common.policies.data.PersistentTopicStats) Producer(org.apache.pulsar.client.api.Producer) LoggerFactory(org.slf4j.LoggerFactory) Assert.assertEquals(org.testng.Assert.assertEquals) CompletableFuture(java.util.concurrent.CompletableFuture) ConsumerBuilder(org.apache.pulsar.client.api.ConsumerBuilder) Test(org.testng.annotations.Test) Message(org.apache.pulsar.client.api.Message) HashSet(java.util.HashSet) Lists(com.google.common.collect.Lists) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CollectionUtils(org.apache.commons.collections.CollectionUtils) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) BlockingArrayQueue(org.eclipse.jetty.util.BlockingArrayQueue) AfterClass(org.testng.annotations.AfterClass) ConsumerImpl(org.apache.pulsar.client.impl.ConsumerImpl) Logger(org.slf4j.Logger) SubType(org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType) Assert.fail(org.testng.Assert.fail) BeforeClass(org.testng.annotations.BeforeClass) Set(java.util.Set) Assert.assertNotNull(org.testng.Assert.assertNotNull) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) TimeUnit(java.util.concurrent.TimeUnit) CountDownLatch(java.util.concurrent.CountDownLatch) Consumer(org.apache.pulsar.client.api.Consumer) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MessageId(org.apache.pulsar.client.api.MessageId) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) Assert.assertTrue(org.testng.Assert.assertTrue) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CountDownLatch(java.util.concurrent.CountDownLatch) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) Test(org.testng.annotations.Test)

Aggregations

List (java.util.List)3 TimeUnit (java.util.concurrent.TimeUnit)3 ConsumerBuilder (org.apache.pulsar.client.api.ConsumerBuilder)3 SubscriptionType (org.apache.pulsar.client.api.SubscriptionType)3 Lists (com.google.common.collect.Lists)2 Map (java.util.Map)2 Properties (java.util.Properties)2 Set (java.util.Set)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 ClientBuilder (org.apache.pulsar.client.api.ClientBuilder)2 Consumer (org.apache.pulsar.client.api.Consumer)2 Message (org.apache.pulsar.client.api.Message)2 MessageId (org.apache.pulsar.client.api.MessageId)2 MessageListener (org.apache.pulsar.client.api.MessageListener)2 PulsarClient (org.apache.pulsar.client.api.PulsarClient)2 PulsarClientException (org.apache.pulsar.client.api.PulsarClientException)2 TopicName (org.apache.pulsar.common.naming.TopicName)2 FutureUtil (org.apache.pulsar.common.util.FutureUtil)2 Logger (org.slf4j.Logger)2 LoggerFactory (org.slf4j.LoggerFactory)2