Search in sources :

Example 1 with ClientBuilder

use of org.apache.pulsar.client.api.ClientBuilder in project incubator-pulsar by apache.

the class BrokerService method getReplicationClient.

public PulsarClient getReplicationClient(String cluster) {
    PulsarClient client = replicationClients.get(cluster);
    if (client != null) {
        return client;
    }
    return replicationClients.computeIfAbsent(cluster, key -> {
        try {
            String path = PulsarWebResource.path("clusters", cluster);
            ClusterData data = this.pulsar.getConfigurationCache().clustersCache().get(path).orElseThrow(() -> new KeeperException.NoNodeException(path));
            ClientBuilder clientBuilder = PulsarClient.builder().enableTcpNoDelay(false).connectionsPerBroker(pulsar.getConfiguration().getReplicationConnectionsPerBroker()).statsInterval(0, TimeUnit.SECONDS);
            if (pulsar.getConfiguration().isAuthenticationEnabled()) {
                clientBuilder.authentication(pulsar.getConfiguration().getBrokerClientAuthenticationPlugin(), pulsar.getConfiguration().getBrokerClientAuthenticationParameters());
            }
            if (pulsar.getConfiguration().isReplicationTlsEnabled()) {
                clientBuilder.serviceUrl(isNotBlank(data.getBrokerServiceUrlTls()) ? data.getBrokerServiceUrlTls() : data.getServiceUrlTls()).enableTls(true).tlsTrustCertsFilePath(pulsar.getConfiguration().getBrokerClientTrustCertsFilePath()).allowTlsInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection());
            } else {
                clientBuilder.serviceUrl(isNotBlank(data.getBrokerServiceUrl()) ? data.getBrokerServiceUrl() : data.getServiceUrl());
            }
            // Share all the IO threads across broker and client connections
            ClientConfigurationData conf = ((ClientBuilderImpl) clientBuilder).getClientConfigurationData();
            return new PulsarClientImpl(conf, workerGroup);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
}
Also used : ClientConfigurationData(org.apache.pulsar.client.impl.conf.ClientConfigurationData) ClientBuilderImpl(org.apache.pulsar.client.impl.ClientBuilderImpl) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) PulsarClient(org.apache.pulsar.client.api.PulsarClient) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) KeeperException(org.apache.zookeeper.KeeperException) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) IOException(java.io.IOException) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) KeeperException(org.apache.zookeeper.KeeperException) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder)

Example 2 with ClientBuilder

use of org.apache.pulsar.client.api.ClientBuilder in project incubator-pulsar by apache.

the class PerformanceReader method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-reader");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
        if (arguments.useTls == false) {
            arguments.useTls = Boolean.parseBoolean(prop.getProperty("useTls"));
        }
        if (isBlank(arguments.tlsTrustCertsFilePath)) {
            arguments.tlsTrustCertsFilePath = prop.getProperty("tlsTrustCertsFilePath", "");
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance reader with config: {}", w.writeValueAsString(arguments));
    final TopicName prefixTopicName = TopicName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    ReaderListener<byte[]> listener = (reader, msg) -> {
        messagesReceived.increment();
        bytesReceived.add(msg.getData().length);
        if (limiter != null) {
            limiter.acquire();
        }
    };
    ClientBuilder clientBuilder = // 
    PulsarClient.builder().serviceUrl(// 
    arguments.serviceURL).connectionsPerBroker(// 
    arguments.maxConnections).statsInterval(arguments.statsIntervalSeconds, // 
    TimeUnit.SECONDS).ioThreads(// 
    Runtime.getRuntime().availableProcessors()).enableTls(// 
    arguments.useTls).tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientBuilder.authentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = clientBuilder.build();
    List<CompletableFuture<Reader<byte[]>>> futures = Lists.newArrayList();
    MessageId startMessageId;
    if ("earliest".equals(arguments.startMessageId)) {
        startMessageId = MessageId.earliest;
    } else if ("latest".equals(arguments.startMessageId)) {
        startMessageId = MessageId.latest;
    } else {
        String[] parts = arguments.startMessageId.split(":");
        startMessageId = new MessageIdImpl(Long.parseLong(parts[0]), Long.parseLong(parts[1]), -1);
    }
    ReaderBuilder<byte[]> readerBuilder = // 
    pulsarClient.newReader().readerListener(// 
    listener).receiverQueueSize(// 
    arguments.receiverQueueSize).startMessageId(startMessageId);
    for (int i = 0; i < arguments.numTopics; i++) {
        final TopicName topicName = (arguments.numTopics == 1) ? prefixTopicName : TopicName.get(String.format("%s-%d", prefixTopicName, i));
        futures.add(readerBuilder.clone().topic(topicName.toString()).createAsync());
    }
    FutureUtil.waitForAll(futures).get();
    log.info("Start reading from {} topics", arguments.numTopics);
    long oldTime = System.nanoTime();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        log.info("Read throughput: {}  msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput));
        oldTime = now;
    }
    pulsarClient.close();
}
Also used : LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) ParameterException(com.beust.jcommander.ParameterException) Parameter(com.beust.jcommander.Parameter) LoggerFactory(org.slf4j.LoggerFactory) CompletableFuture(java.util.concurrent.CompletableFuture) RateLimiter(com.google.common.util.concurrent.RateLimiter) ReaderListener(org.apache.pulsar.client.api.ReaderListener) Lists(com.google.common.collect.Lists) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Properties(java.util.Properties) Logger(org.slf4j.Logger) ObjectWriter(com.fasterxml.jackson.databind.ObjectWriter) DecimalFormat(java.text.DecimalFormat) JCommander(com.beust.jcommander.JCommander) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Reader(org.apache.pulsar.client.api.Reader) FileInputStream(java.io.FileInputStream) TimeUnit(java.util.concurrent.TimeUnit) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) MessageId(org.apache.pulsar.client.api.MessageId) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) ReaderBuilder(org.apache.pulsar.client.api.ReaderBuilder) ObjectWriter(com.fasterxml.jackson.databind.ObjectWriter) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) Properties(java.util.Properties) FileInputStream(java.io.FileInputStream) RateLimiter(com.google.common.util.concurrent.RateLimiter) TopicName(org.apache.pulsar.common.naming.TopicName) CompletableFuture(java.util.concurrent.CompletableFuture) JCommander(com.beust.jcommander.JCommander) ParameterException(com.beust.jcommander.ParameterException) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) MessageId(org.apache.pulsar.client.api.MessageId)

Example 3 with ClientBuilder

use of org.apache.pulsar.client.api.ClientBuilder in project incubator-pulsar by apache.

the class PerformanceConsumer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-consumer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
        if (arguments.useTls == false) {
            arguments.useTls = Boolean.parseBoolean(prop.getProperty("useTls"));
        }
        if (isBlank(arguments.tlsTrustCertsFilePath)) {
            arguments.tlsTrustCertsFilePath = prop.getProperty("tlsTrustCertsFilePath", "");
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments));
    final TopicName prefixTopicName = TopicName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    MessageListener<byte[]> listener = (consumer, msg) -> {
        messagesReceived.increment();
        bytesReceived.add(msg.getData().length);
        if (limiter != null) {
            limiter.acquire();
        }
        long latencyMillis = System.currentTimeMillis() - msg.getPublishTime();
        if (latencyMillis >= 0) {
            recorder.recordValue(latencyMillis);
            cumulativeRecorder.recordValue(latencyMillis);
        }
        consumer.acknowledgeAsync(msg);
    };
    ClientBuilder clientBuilder = // 
    PulsarClient.builder().serviceUrl(// 
    arguments.serviceURL).connectionsPerBroker(// 
    arguments.maxConnections).statsInterval(arguments.statsIntervalSeconds, // 
    TimeUnit.SECONDS).ioThreads(// 
    Runtime.getRuntime().availableProcessors()).enableTls(// 
    arguments.useTls).tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientBuilder.authentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = clientBuilder.build();
    class EncKeyReader implements CryptoKeyReader {

        EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();

        EncKeyReader(byte[] value) {
            keyInfo.setKey(value);
        }

        @Override
        public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
            return null;
        }

        @Override
        public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
            if (keyName.equals(arguments.encKeyName)) {
                return keyInfo;
            }
            return null;
        }
    }
    List<Future<Consumer<byte[]>>> futures = Lists.newArrayList();
    ConsumerBuilder<byte[]> consumerBuilder = // 
    pulsarClient.newConsumer().messageListener(// 
    listener).receiverQueueSize(// 
    arguments.receiverQueueSize).subscriptionType(arguments.subscriptionType);
    if (arguments.encKeyName != null) {
        byte[] pKey = Files.readAllBytes(Paths.get(arguments.encKeyFile));
        EncKeyReader keyReader = new EncKeyReader(pKey);
        consumerBuilder.cryptoKeyReader(keyReader);
    }
    for (int i = 0; i < arguments.numTopics; i++) {
        final TopicName topicName = (arguments.numTopics == 1) ? prefixTopicName : TopicName.get(String.format("%s-%d", prefixTopicName, i));
        log.info("Adding {} consumers on topic {}", arguments.numConsumers, topicName);
        for (int j = 0; j < arguments.numConsumers; j++) {
            String subscriberName;
            if (arguments.numConsumers > 1) {
                subscriberName = String.format("%s-%d", arguments.subscriberName, j);
            } else {
                subscriberName = arguments.subscriberName;
            }
            futures.add(consumerBuilder.clone().topic(topicName.toString()).subscriptionName(subscriberName).subscribeAsync());
        }
    }
    for (Future<Consumer<byte[]>> future : futures) {
        future.get();
    }
    log.info("Start receiving from {} consumers on {} topics", arguments.numConsumers, arguments.numTopics);
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput received: {}  msg/s -- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", dec.format(rate), dec.format(throughput), dec.format(reportHistogram.getMean()), (long) reportHistogram.getValueAtPercentile(50), (long) reportHistogram.getValueAtPercentile(95), (long) reportHistogram.getValueAtPercentile(99), (long) reportHistogram.getValueAtPercentile(99.9), (long) reportHistogram.getValueAtPercentile(99.99), (long) reportHistogram.getMaxValue());
        reportHistogram.reset();
        oldTime = now;
    }
    pulsarClient.close();
}
Also used : LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) ParameterException(com.beust.jcommander.ParameterException) Parameter(com.beust.jcommander.Parameter) LoggerFactory(org.slf4j.LoggerFactory) ConsumerBuilder(org.apache.pulsar.client.api.ConsumerBuilder) RateLimiter(com.google.common.util.concurrent.RateLimiter) Future(java.util.concurrent.Future) Lists(com.google.common.collect.Lists) Map(java.util.Map) Recorder(org.HdrHistogram.Recorder) EncryptionKeyInfo(org.apache.pulsar.client.api.EncryptionKeyInfo) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Properties(java.util.Properties) Logger(org.slf4j.Logger) Files(java.nio.file.Files) ObjectWriter(com.fasterxml.jackson.databind.ObjectWriter) MessageListener(org.apache.pulsar.client.api.MessageListener) DecimalFormat(java.text.DecimalFormat) JCommander(com.beust.jcommander.JCommander) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) FileInputStream(java.io.FileInputStream) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) TimeUnit(java.util.concurrent.TimeUnit) Histogram(org.HdrHistogram.Histogram) Consumer(org.apache.pulsar.client.api.Consumer) CryptoKeyReader(org.apache.pulsar.client.api.CryptoKeyReader) List(java.util.List) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Paths(java.nio.file.Paths) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Histogram(org.HdrHistogram.Histogram) EncryptionKeyInfo(org.apache.pulsar.client.api.EncryptionKeyInfo) Properties(java.util.Properties) CryptoKeyReader(org.apache.pulsar.client.api.CryptoKeyReader) Consumer(org.apache.pulsar.client.api.Consumer) JCommander(com.beust.jcommander.JCommander) ParameterException(com.beust.jcommander.ParameterException) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) ObjectWriter(com.fasterxml.jackson.databind.ObjectWriter) FileInputStream(java.io.FileInputStream) RateLimiter(com.google.common.util.concurrent.RateLimiter) TopicName(org.apache.pulsar.common.naming.TopicName) Future(java.util.concurrent.Future) Map(java.util.Map)

Example 4 with ClientBuilder

use of org.apache.pulsar.client.api.ClientBuilder in project incubator-pulsar by apache.

the class PerformanceProducer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topics.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
        if (arguments.useTls == false) {
            arguments.useTls = Boolean.parseBoolean(prop.getProperty("useTls"));
        }
        if (isBlank(arguments.tlsTrustCertsFilePath)) {
            arguments.tlsTrustCertsFilePath = prop.getProperty("tlsTrustCertsFilePath", "");
        }
    }
    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments));
    // Read payload data from file if needed
    byte[] payloadData;
    if (arguments.payloadFilename != null) {
        payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename));
    } else {
        payloadData = new byte[arguments.msgSize];
    }
    // Now processing command line arguments
    String prefixTopicName = arguments.topics.get(0);
    List<Future<Producer<byte[]>>> futures = Lists.newArrayList();
    ClientBuilder clientBuilder = // 
    PulsarClient.builder().serviceUrl(// 
    arguments.serviceURL).connectionsPerBroker(// 
    arguments.maxConnections).ioThreads(// 
    Runtime.getRuntime().availableProcessors()).statsInterval(arguments.statsIntervalSeconds, // 
    TimeUnit.SECONDS).enableTls(// 
    arguments.useTls).tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientBuilder.authentication(arguments.authPluginClassName, arguments.authParams);
    }
    class EncKeyReader implements CryptoKeyReader {

        EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();

        EncKeyReader(byte[] value) {
            keyInfo.setKey(value);
        }

        @Override
        public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
            if (keyName.equals(arguments.encKeyName)) {
                return keyInfo;
            }
            return null;
        }

        @Override
        public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
            return null;
        }
    }
    PulsarClient client = clientBuilder.build();
    ProducerBuilder<byte[]> producerBuilder = // 
    client.newProducer().sendTimeout(0, // 
    TimeUnit.SECONDS).compressionType(// 
    arguments.compression).maxPendingMessages(// 
    arguments.maxOutstanding).messageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    if (arguments.batchTime > 0) {
        producerBuilder.batchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS).enableBatching(true);
    }
    // Block if queue is full else we will start seeing errors in sendAsync
    producerBuilder.blockIfQueueFull(true);
    if (arguments.encKeyName != null) {
        producerBuilder.addEncryptionKey(arguments.encKeyName);
        byte[] pKey = Files.readAllBytes(Paths.get(arguments.encKeyFile));
        EncKeyReader keyReader = new EncKeyReader(pKey);
        producerBuilder.cryptoKeyReader(keyReader);
    }
    for (int i = 0; i < arguments.numTopics; i++) {
        String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i);
        log.info("Adding {} publishers on topic {}", arguments.numProducers, topic);
        for (int j = 0; j < arguments.numProducers; j++) {
            futures.add(producerBuilder.clone().topic(topic).createAsync());
        }
    }
    final List<Producer<byte[]>> producers = Lists.newArrayListWithCapacity(futures.size());
    for (Future<Producer<byte[]>> future : futures) {
        producers.add(future.get());
    }
    log.info("Created {} producers", producers.size());
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    Collections.shuffle(producers);
    AtomicBoolean isDone = new AtomicBoolean();
    executor.submit(() -> {
        try {
            RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate);
            long startTime = System.currentTimeMillis();
            // Send messages on all topics/producers
            long totalSent = 0;
            while (true) {
                for (Producer<byte[]> producer : producers) {
                    if (arguments.testTime > 0) {
                        if (System.currentTimeMillis() - startTime > arguments.testTime) {
                            log.info("------------------- DONE -----------------------");
                            printAggregatedStats();
                            isDone.set(true);
                            Thread.sleep(5000);
                            System.exit(0);
                        }
                    }
                    if (arguments.numMessages > 0) {
                        if (totalSent++ >= arguments.numMessages) {
                            log.info("------------------- DONE -----------------------");
                            printAggregatedStats();
                            isDone.set(true);
                            Thread.sleep(5000);
                            System.exit(0);
                        }
                    }
                    rateLimiter.acquire();
                    final long sendTime = System.nanoTime();
                    producer.sendAsync(payloadData).thenRun(() -> {
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);
                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);
                    }).exceptionally(ex -> {
                        log.warn("Write error on message", ex);
                        System.exit(-1);
                        return null;
                    });
                }
            }
        } catch (Throwable t) {
            log.error("Got error", t);
        }
    });
    // Print report stats
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm";
    log.info("Dumping latency stats to {}", statsFileName);
    PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
    HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog);
    // Some log header bits
    histogramLogWriter.outputLogFormatVersion();
    histogramLogWriter.outputLegend();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        if (isDone.get()) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0));
        histogramLogWriter.outputIntervalHistogram(reportHistogram);
        reportHistogram.reset();
        oldTime = now;
    }
    client.close();
}
Also used : Histogram(org.HdrHistogram.Histogram) EncryptionKeyInfo(org.apache.pulsar.client.api.EncryptionKeyInfo) Properties(java.util.Properties) CryptoKeyReader(org.apache.pulsar.client.api.CryptoKeyReader) JCommander(com.beust.jcommander.JCommander) ParameterException(com.beust.jcommander.ParameterException) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) HistogramLogWriter(org.HdrHistogram.HistogramLogWriter) PrintStream(java.io.PrintStream) ObjectWriter(com.fasterxml.jackson.databind.ObjectWriter) FileInputStream(java.io.FileInputStream) RateLimiter(com.google.common.util.concurrent.RateLimiter) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Producer(org.apache.pulsar.client.api.Producer) FileOutputStream(java.io.FileOutputStream) Future(java.util.concurrent.Future) Map(java.util.Map)

Example 5 with ClientBuilder

use of org.apache.pulsar.client.api.ClientBuilder in project incubator-pulsar by apache.

the class PulsarClientKafkaConfig method getClientBuilder.

public static ClientBuilder getClientBuilder(Properties properties) {
    ClientBuilder clientBuilder = PulsarClient.builder();
    if (properties.containsKey(AUTHENTICATION_CLASS)) {
        String className = properties.getProperty(AUTHENTICATION_CLASS);
        try {
            @SuppressWarnings("unchecked") Class<Authentication> clazz = (Class<Authentication>) Class.forName(className);
            Authentication auth = clazz.newInstance();
            clientBuilder.authentication(auth);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    clientBuilder.enableTls(Boolean.parseBoolean(properties.getProperty(USE_TLS, "false")));
    clientBuilder.allowTlsInsecureConnection(Boolean.parseBoolean(properties.getProperty(TLS_ALLOW_INSECURE_CONNECTION, "false")));
    clientBuilder.enableTlsHostnameVerification(Boolean.parseBoolean(properties.getProperty(TLS_HOSTNAME_VERIFICATION, "false")));
    if (properties.containsKey(TLS_TRUST_CERTS_FILE_PATH)) {
        clientBuilder.tlsTrustCertsFilePath(properties.getProperty(TLS_TRUST_CERTS_FILE_PATH));
    }
    if (properties.containsKey(OPERATION_TIMEOUT_MS)) {
        clientBuilder.operationTimeout(Integer.parseInt(properties.getProperty(OPERATION_TIMEOUT_MS)), TimeUnit.MILLISECONDS);
    }
    if (properties.containsKey(STATS_INTERVAL_SECONDS)) {
        clientBuilder.statsInterval(Integer.parseInt(properties.getProperty(STATS_INTERVAL_SECONDS)), TimeUnit.SECONDS);
    }
    if (properties.containsKey(NUM_IO_THREADS)) {
        clientBuilder.ioThreads(Integer.parseInt(properties.getProperty(NUM_IO_THREADS)));
    }
    if (properties.containsKey(CONNECTIONS_PER_BROKER)) {
        clientBuilder.connectionsPerBroker(Integer.parseInt(properties.getProperty(CONNECTIONS_PER_BROKER)));
    }
    if (properties.containsKey(USE_TCP_NODELAY)) {
        clientBuilder.enableTcpNoDelay(Boolean.parseBoolean(properties.getProperty(USE_TCP_NODELAY)));
    }
    if (properties.containsKey(CONCURRENT_LOOKUP_REQUESTS)) {
        clientBuilder.maxConcurrentLookupRequests(Integer.parseInt(properties.getProperty(CONCURRENT_LOOKUP_REQUESTS)));
    }
    if (properties.containsKey(MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION)) {
        clientBuilder.maxNumberOfRejectedRequestPerConnection(Integer.parseInt(properties.getProperty(MAX_NUMBER_OF_REJECTED_REQUESTS_PER_CONNECTION)));
    }
    return clientBuilder;
}
Also used : Authentication(org.apache.pulsar.client.api.Authentication) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder)

Aggregations

ClientBuilder (org.apache.pulsar.client.api.ClientBuilder)6 PulsarClient (org.apache.pulsar.client.api.PulsarClient)4 JCommander (com.beust.jcommander.JCommander)3 ParameterException (com.beust.jcommander.ParameterException)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)3 ObjectWriter (com.fasterxml.jackson.databind.ObjectWriter)3 RateLimiter (com.google.common.util.concurrent.RateLimiter)3 FileInputStream (java.io.FileInputStream)3 Properties (java.util.Properties)3 Parameter (com.beust.jcommander.Parameter)2 Lists (com.google.common.collect.Lists)2 DecimalFormat (java.text.DecimalFormat)2 List (java.util.List)2 Map (java.util.Map)2 Future (java.util.concurrent.Future)2 TimeUnit (java.util.concurrent.TimeUnit)2 LongAdder (java.util.concurrent.atomic.LongAdder)2 Histogram (org.HdrHistogram.Histogram)2 StringUtils.isBlank (org.apache.commons.lang3.StringUtils.isBlank)2 StringUtils.isNotBlank (org.apache.commons.lang3.StringUtils.isNotBlank)2