Examples with Producer com.yahoo.pulsar.client.api.Producer used on opensource projects

Example 21 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class UnAcknowledgedMessagesTimeoutTest method testExclusiveCumulativeAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testExclusiveCumulativeAckedNormalTopic() throws Exception {
    String key = "testExclusiveCumulativeAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(7);
    conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message
    HashSet<String> hSet = new HashSet<>();
    Message message = consumer.receive();
    Message lastMessage = message;
    while (message != null) {
        lastMessage = message;
        hSet.add(new String(message.getData()));
        log.info("Consumer received " + new String(message.getData()));
        log.info("Message ID details " + ((MessageIdImpl) message.getMessageId()).toString());
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    }
    long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    assertEquals(size, totalMessages);
    log.info("Comulative Ack sent for " + new String(lastMessage.getData()));
    log.info("Message ID details " + ((MessageIdImpl) lastMessage.getMessageId()).toString());
    consumer.acknowledgeCumulative(lastMessage);
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    assertEquals(size, 0);
    message = consumer.receive((int) (2 * ackTimeOutMillis), TimeUnit.MILLISECONDS);
    assertEquals(message, null);
}

Example 22 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ClientErrorsTest method producerCreateSuccessAfterRetry.

private void producerCreateSuccessAfterRetry(String topic) throws Exception {
    PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
    final AtomicInteger counter = new AtomicInteger(0);
    mockBrokerService.setHandleProducer((ctx, producer) -> {
        if (counter.incrementAndGet() == 2) {
            ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer"));
            return;
        }
        ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.ServiceNotReady, "msg"));
    });
    try {
        Producer producer = client.createProducer(topic);
    } catch (Exception e) {
        fail("Should not fail");
    }
    mockBrokerService.resetHandleProducer();
    client.close();
}

Example 23 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ClientErrorsTest method testOneProducerFailShouldCloseAllProducersInPartitionedProducer.

// if a producer fails to connect while creating partitioned producer, it should close all successful connections of
// other producers and fail
@Test
public void testOneProducerFailShouldCloseAllProducersInPartitionedProducer() throws Exception {
    PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
    final AtomicInteger producerCounter = new AtomicInteger(0);
    final AtomicInteger closeCounter = new AtomicInteger(0);
    mockBrokerService.setHandleProducer((ctx, producer) -> {
        if (producerCounter.incrementAndGet() == 3) {
            ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.AuthorizationError, "msg"));
            return;
        }
        ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer"));
    });
    mockBrokerService.setHandleCloseProducer((ctx, closeProducer) -> {
        ctx.writeAndFlush(Commands.newSuccess(closeProducer.getRequestId()));
        closeCounter.incrementAndGet();
    });
    try {
        Producer producer = client.createProducer("persistent://prop/use/ns/multi-part-t1");
        fail("Should have failed with an authorization error");
    } catch (Exception e) {
        assertTrue(e instanceof PulsarClientException.AuthorizationException);
        // should call close for 3 partitions
        assertEquals(closeCounter.get(), 3);
    }
    mockBrokerService.resetHandleProducer();
    mockBrokerService.resetHandleCloseProducer();
    client.close();
}

Example 24 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PerformanceProducer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.destinations.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
    }
    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments));
    // Read payload data from file if needed
    byte[] payloadData;
    if (arguments.payloadFilename != null) {
        payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename));
    } else {
        payloadData = new byte[arguments.msgSize];
    }
    // Now processing command line arguments
    String prefixTopicName = arguments.destinations.get(0);
    List<Future<Producer>> futures = Lists.newArrayList();
    EventLoopGroup eventLoopGroup;
    if (SystemUtils.IS_OS_LINUX) {
        eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer"));
    } else {
        eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer"));
    }
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionsPerBroker(arguments.maxConnections);
    clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setSendTimeout(0, TimeUnit.SECONDS);
    producerConf.setCompressionType(arguments.compression);
    // enable round robin message routing if it is a partitioned topic
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    if (arguments.batchTime > 0) {
        producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS);
        producerConf.setBatchingEnabled(true);
        producerConf.setMaxPendingMessages(arguments.msgRate);
    }
    for (int i = 0; i < arguments.numTopics; i++) {
        String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i);
        log.info("Adding {} publishers on destination {}", arguments.numProducers, topic);
        for (int j = 0; j < arguments.numProducers; j++) {
            futures.add(client.createProducerAsync(topic, producerConf));
        }
    }
    final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size());
    for (Future<Producer> future : futures) {
        producers.add(future.get());
    }
    log.info("Created {} producers", producers.size());
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    Collections.shuffle(producers);
    AtomicBoolean isDone = new AtomicBoolean();
    executor.submit(() -> {
        try {
            RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate);
            long startTime = System.currentTimeMillis();
            // Send messages on all topics/producers
            long totalSent = 0;
            while (true) {
                for (Producer producer : producers) {
                    if (arguments.testTime > 0) {
                        if (System.currentTimeMillis() - startTime > arguments.testTime) {
                            log.info("------------------- DONE -----------------------");
                            printAggregatedStats();
                            isDone.set(true);
                            Thread.sleep(5000);
                            System.exit(0);
                        }
                    }
                    if (arguments.numMessages > 0) {
                        if (totalSent++ >= arguments.numMessages) {
                            log.info("------------------- DONE -----------------------");
                            printAggregatedStats();
                            isDone.set(true);
                            Thread.sleep(5000);
                            System.exit(0);
                        }
                    }
                    rateLimiter.acquire();
                    final long sendTime = System.nanoTime();
                    producer.sendAsync(payloadData).thenRun(() -> {
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);
                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);
                    }).exceptionally(ex -> {
                        log.warn("Write error on message", ex);
                        System.exit(-1);
                        return null;
                    });
                }
            }
        } catch (Throwable t) {
            log.error("Got error", t);
        }
    });
    // Print report stats
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm";
    log.info("Dumping latency stats to {}", statsFileName);
    PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
    HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog);
    // Some log header bits
    histogramLogWriter.outputLogFormatVersion();
    histogramLogWriter.outputLegend();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        if (isDone.get()) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0));
        histogramLogWriter.outputIntervalHistogram(reportHistogram);
        reportHistogram.reset();
        oldTime = now;
    }
    client.close();
}

Example 25 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PersistentQueueE2ETest method testRoundRobinBatchDistribution.

// this test is good to have to see the distribution, but every now and then it gets slightly different than the
// expected numbers. keeping this disabled to not break the build, but nevertheless this gives good insight into
// how the round robin distribution algorithm is behaving
@Test(enabled = false)
public void testRoundRobinBatchDistribution() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic5";
    final String subName = "sub5";
    final int numMsgs = 137;
    /* some random number different than default batch size of 100 */
    final AtomicInteger counter1 = new AtomicInteger(0);
    final AtomicInteger counter2 = new AtomicInteger(0);
    final AtomicInteger counter3 = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(numMsgs * 3);
    ConsumerConfiguration conf1 = new ConsumerConfiguration();
    conf1.setSubscriptionType(SubscriptionType.Shared);
    conf1.setReceiverQueueSize(10);
    conf1.setMessageListener((consumer, msg) -> {
        try {
            counter1.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    });
    ConsumerConfiguration conf2 = new ConsumerConfiguration();
    conf2.setSubscriptionType(SubscriptionType.Shared);
    conf2.setReceiverQueueSize(10);
    conf2.setMessageListener((consumer, msg) -> {
        try {
            counter2.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    });
    ConsumerConfiguration conf3 = new ConsumerConfiguration();
    conf3.setSubscriptionType(SubscriptionType.Shared);
    conf3.setReceiverQueueSize(10);
    conf3.setMessageListener((consumer, msg) -> {
        try {
            counter3.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    });
    // subscribe and close, so that distribution can be checked after
    // all messages are published
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, conf1);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, conf2);
    Consumer consumer3 = pulsarClient.subscribe(topicName, subName, conf3);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer producer = pulsarClient.createProducer(topicName);
    for (int i = 0; i < numMsgs * 3; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    latch.await(1, TimeUnit.SECONDS);
    /*
         * total messages = 137 * 3 = 411 Each consumer has 10 permits. There will be 411 / 3*10 = 13 full distributions
         * i.e. each consumer will get 130 messages. In the 14th round, the balance is 411 - 130*3 = 21. Two consumers
         * will get another batch of 10 messages (Total: 140) and the 3rd one will get the last one (Total: 131)
         */
    assertTrue(CollectionUtils.subtract(Lists.newArrayList(140, 140, 131), Lists.newArrayList(counter1.get(), counter2.get(), counter3.get())).isEmpty());
    consumer1.close();
    consumer2.close();
    consumer3.close();
    admin.persistentTopics().delete(topicName);
}