Examples with ProducerConfiguration com.yahoo.pulsar.client.api.ProducerConfiguration used on opensource projects

Example 6 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class PersistentTopicE2ETest method testCompression.

@Test(dataProvider = "codec")
public void testCompression(CompressionType compressionType) throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic0" + compressionType;
    // 1. producer connect
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. producer publish messages
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (int i = 0; i < 10; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        assertEquals(msg.getData(), ("my-message-" + i).getBytes());
    }
    // 3. producer disconnect
    producer.close();
    consumer.close();
}

Example 7 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class PersistentTopicE2ETest method testProducerQueueFullBlocking.

@Test
public void testProducerQueueFullBlocking() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-xyzx";
    final int messages = 10;
    PulsarClient client = PulsarClient.create(brokerUrl.toString());
    // 1. Producer connect
    ProducerConfiguration producerConfiguration = new ProducerConfiguration().setMaxPendingMessages(messages).setBlockIfQueueFull(true).setSendTimeout(1, TimeUnit.SECONDS);
    ProducerImpl producer = (ProducerImpl) client.createProducer(topicName, producerConfiguration);
    // 2. Stop broker
    cleanup();
    // 2. producer publish messages
    long startTime = System.nanoTime();
    for (int i = 0; i < messages; i++) {
        // Should never block
        producer.sendAsync("msg".getBytes());
    }
    // Verify thread was not blocked
    long delayNs = System.nanoTime() - startTime;
    assertTrue(delayNs < TimeUnit.SECONDS.toNanos(1));
    assertEquals(producer.getPendingQueueSize(), messages);
    // Next send operation must block, until all the messages in the queue expire
    startTime = System.nanoTime();
    producer.sendAsync("msg".getBytes());
    delayNs = System.nanoTime() - startTime;
    assertTrue(delayNs > TimeUnit.MILLISECONDS.toNanos(500));
    assertTrue(delayNs < TimeUnit.MILLISECONDS.toNanos(1500));
    assertEquals(producer.getPendingQueueSize(), 1);
    // 4. producer disconnect
    producer.close();
    client.close();
    // 5. Restart broker
    setup();
}

Example 8 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class PersistentTopicE2ETest method testBrokerTopicStats.

@Test
public void testBrokerTopicStats() throws Exception {
    BrokerService brokerService = this.pulsar.getBrokerService();
    Field field = BrokerService.class.getDeclaredField("statsUpdater");
    field.setAccessible(true);
    ScheduledExecutorService statsUpdater = (ScheduledExecutorService) field.get(brokerService);
    // disable statsUpdate to calculate rates explicitly
    statsUpdater.shutdown();
    final String namespace = "prop/use/ns-abc";
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://" + namespace + "/topic0", producerConf);
    // 1. producer publish messages
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Metrics metric = null;
    // sleep 1 sec to caclulate metrics per second
    Thread.sleep(1000);
    brokerService.updateRates();
    List<Metrics> metrics = brokerService.getDestinationMetrics();
    for (int i = 0; i < metrics.size(); i++) {
        if (metrics.get(i).getDimension("namespace").equalsIgnoreCase(namespace)) {
            metric = metrics.get(i);
            break;
        }
    }
    assertNotNull(metric);
    double msgInRate = (double) metrics.get(0).getMetrics().get("brk_in_rate");
    // rate should be calculated and no must be > 0 as we have produced 10 msgs so far
    assertTrue(msgInRate > 0);
}

Example 9 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerConsumer.

@Test(dataProvider = "codec")
public void testSimpleBatchProducerConsumer(CompressionType compressionType) throws Exception {
    int numMsgs = 500;
    int numMsgsInBatch = numMsgs / 20;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer";
    final String subscriptionName = "pc-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        if (i % 2 == 0) {
            consumer.acknowledgeCumulative(msg);
        } else {
            lastunackedMsg = msg;
        }
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}

Example 10 with ProducerConfiguration

use of com.yahoo.pulsar.client.api.ProducerConfiguration in project pulsar by yahoo.

the class ZeroQueueSizeTest method testFailedZeroQueueSizeBatchMessage.

@Test()
public void testFailedZeroQueueSizeBatchMessage() throws PulsarClientException {
    int batchMessageDelayMs = 100;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(0);
    Consumer consumer = pulsarClient.subscribe("persistent://prop-xyz/use/ns-abc/topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns-abc/topic1", producerConf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    try {
        consumer.receiveAsync().handle((ok, e) -> {
            if (e == null) {
                // as zero receiverQueueSize doesn't support batch message, must receive exception at callback.
                Assert.fail();
            }
            return null;
        });
    } finally {
        consumer.close();
    }
}