Examples with ProducerConfiguration com.yahoo.pulsar.client.api.ProducerConfiguration used on opensource projects

Example 11 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class BrokerClientIntegrationTest method testCloseBrokerService.

/**
     * Verifies: 1. Closing of Broker service unloads all bundle gracefully and there must not be any connected bundles
     * after closing broker service
     * 
     * @throws Exception
     */
@Test
public void testCloseBrokerService() throws Exception {
    final String ns1 = "my-property/use/brok-ns1";
    final String ns2 = "my-property/use/brok-ns2";
    admin.namespaces().createNamespace(ns1);
    admin.namespaces().createNamespace(ns2);
    final String dn1 = "persistent://" + ns1 + "/my-topic";
    final String dn2 = "persistent://" + ns2 + "/my-topic";
    ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(dn1, "my-subscriber-name", new ConsumerConfiguration());
    ProducerImpl producer1 = (ProducerImpl) pulsarClient.createProducer(dn1, new ProducerConfiguration());
    ProducerImpl producer2 = (ProducerImpl) pulsarClient.createProducer(dn2, new ProducerConfiguration());
    // unload all other namespace
    pulsar.getBrokerService().close();
    // [1] OwnershipCache should not contain any more namespaces
    OwnershipCache ownershipCache = pulsar.getNamespaceService().getOwnershipCache();
    assertTrue(ownershipCache.getOwnedBundles().keySet().isEmpty());
    // [2] All clients must be disconnected and in connecting state
    // producer1 must not be able to connect again
    assertTrue(producer1.getClientCnx() == null);
    assertTrue(producer1.getState().equals(State.Connecting));
    // consumer1 must not be able to connect again
    assertTrue(consumer1.getClientCnx() == null);
    assertTrue(consumer1.getState().equals(State.Connecting));
    // producer2 must not be able to connect again
    assertTrue(producer2.getClientCnx() == null);
    assertTrue(producer2.getState().equals(State.Connecting));
    producer1.close();
    producer2.close();
    consumer1.close();
}

Example 12 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class PerMessageUnAcknowledgedRedeliveryTest method testSharedAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedAckedPartitionedTopic() throws Exception {
    String key = "testSharedAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 15;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(50);
    conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message, doesn't ack
    Message message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    long size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    // 5. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 6. Receiver receives the message, ack them
    message = consumer.receive();
    int received = 0;
    while (message != null) {
        received++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    assertEquals(received, 5);
    // 7. Simulate ackTimeout
    ((PartitionedConsumerImpl) consumer).getConsumers().forEach(c -> c.getUnAckedMessageTracker().toggle());
    // 8. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 9. Receiver receives the message, doesn't ack
    message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 10);
    Thread.sleep(ackTimeOutMillis);
    // 10. Receiver receives redelivered messages
    message = consumer.receive();
    int redelivered = 0;
    while (message != null) {
        redelivered++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    assertEquals(redelivered, 5);
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
}

Example 13 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class UnAcknowledgedMessagesTimeoutTest method testSharedSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
    String key = "testSharedSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 20;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(100);
    consumerConfig.setSubscriptionType(SubscriptionType.Shared);
    consumerConfig.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    consumerConfig.setConsumerName("Consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    consumerConfig.setConsumerName("Consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        MessageId msgId = producer.send(message.getBytes());
        log.info("Message produced: {} -- msgId: {}", message, msgId);
    }
    // 4. Receive messages
    int messageCount1 = receiveAllMessage(consumer1, false);
    int messageCount2 = receiveAllMessage(consumer2, true);
    int ackCount1 = 0;
    int ackCount2 = messageCount2;
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    // 5. Check if Messages redelivered again
    // Since receive is a blocking call hoping that timeout will kick in
    Thread.sleep((int) (ackTimeOutMillis * 1.1));
    log.info(key + " Timeout should be triggered now");
    messageCount1 = receiveAllMessage(consumer1, true);
    messageCount2 += receiveAllMessage(consumer2, false);
    ackCount1 = messageCount1;
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    assertEquals(ackCount1 + messageCount2, totalMessages);
    Thread.sleep((int) (ackTimeOutMillis * 1.1));
    // Since receive is a blocking call hoping that timeout will kick in
    log.info(key + " Timeout should be triggered again");
    ackCount1 += receiveAllMessage(consumer1, true);
    ackCount2 += receiveAllMessage(consumer2, true);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(ackCount1 + ackCount2, totalMessages);
}

Example 14 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class PerformanceProducer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.destinations.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
    }
    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments));
    // Read payload data from file if needed
    byte[] payloadData;
    if (arguments.payloadFilename != null) {
        payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename));
    } else {
        payloadData = new byte[arguments.msgSize];
    }
    // Now processing command line arguments
    String prefixTopicName = arguments.destinations.get(0);
    List<Future<Producer>> futures = Lists.newArrayList();
    EventLoopGroup eventLoopGroup;
    if (SystemUtils.IS_OS_LINUX) {
        eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer"));
    } else {
        eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer"));
    }
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionsPerBroker(arguments.maxConnections);
    clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setSendTimeout(0, TimeUnit.SECONDS);
    producerConf.setCompressionType(arguments.compression);
    // enable round robin message routing if it is a partitioned topic
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    if (arguments.batchTime > 0) {
        producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS);
        producerConf.setBatchingEnabled(true);
        producerConf.setMaxPendingMessages(arguments.msgRate);
    }
    for (int i = 0; i < arguments.numTopics; i++) {
        String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i);
        log.info("Adding {} publishers on destination {}", arguments.numProducers, topic);
        for (int j = 0; j < arguments.numProducers; j++) {
            futures.add(client.createProducerAsync(topic, producerConf));
        }
    }
    final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size());
    for (Future<Producer> future : futures) {
        producers.add(future.get());
    }
    log.info("Created {} producers", producers.size());
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    Collections.shuffle(producers);
    AtomicBoolean isDone = new AtomicBoolean();
    executor.submit(() -> {
        try {
            RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate);
            long startTime = System.currentTimeMillis();
            // Send messages on all topics/producers
            long totalSent = 0;
            while (true) {
                for (Producer producer : producers) {
                    if (arguments.testTime > 0) {
                        if (System.currentTimeMillis() - startTime > arguments.testTime) {
                            log.info("------------------- DONE -----------------------");
                            printAggregatedStats();
                            isDone.set(true);
                            Thread.sleep(5000);
                            System.exit(0);
                        }
                    }
                    if (arguments.numMessages > 0) {
                        if (totalSent++ >= arguments.numMessages) {
                            log.info("------------------- DONE -----------------------");
                            printAggregatedStats();
                            isDone.set(true);
                            Thread.sleep(5000);
                            System.exit(0);
                        }
                    }
                    rateLimiter.acquire();
                    final long sendTime = System.nanoTime();
                    producer.sendAsync(payloadData).thenRun(() -> {
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);
                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);
                    }).exceptionally(ex -> {
                        log.warn("Write error on message", ex);
                        System.exit(-1);
                        return null;
                    });
                }
            }
        } catch (Throwable t) {
            log.error("Got error", t);
        }
    });
    // Print report stats
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm";
    log.info("Dumping latency stats to {}", statsFileName);
    PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
    HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog);
    // Some log header bits
    histogramLogWriter.outputLogFormatVersion();
    histogramLogWriter.outputLegend();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        if (isDone.get()) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0));
        histogramLogWriter.outputIntervalHistogram(reportHistogram);
        reportHistogram.reset();
        oldTime = now;
    }
    client.close();
}

Example 15 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class ProducerHandler method onWebSocketConnect.

@Override
public void onWebSocketConnect(Session session) {
    super.onWebSocketConnect(session);
    try {
        ProducerConfiguration conf = getProducerConfiguration();
        producer = service.getPulsarClient().createProducer(topic, conf);
    } catch (Exception e) {
        close(FailedToCreateProducer, e.getMessage());
    }
}