Search in sources :

Example 36 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testProducerExceptionAndThenUnblock.

@Test
public void testProducerExceptionAndThenUnblock() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_exception));
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/quotahold/exceptandunblock";
    final String subName1 = "c1except";
    boolean gotException = false;
    Consumer consumer = client.subscribe(topic1, subName1);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
    byte[] content = new byte[1024];
    Producer producer = client.createProducer(topic1, producerConfiguration);
    for (int i = 0; i < 10; i++) {
        producer.send(content);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    try {
        // try to send over backlog quota and make sure it fails
        producer.send(content);
        producer.send(content);
        Assert.fail("backlog quota did not exceed");
    } catch (PulsarClientException ce) {
        Assert.assertTrue(ce instanceof PulsarClientException.ProducerBlockedQuotaExceededException || ce instanceof PulsarClientException.TimeoutException, ce.getMessage());
        gotException = true;
    }
    Assert.assertTrue(gotException, "backlog exceeded exception did not occur");
    // now remove backlog and ensure that producer is unblockedrolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    int backlog = (int) stats.subscriptions.get(subName1).msgBacklog;
    for (int i = 0; i < backlog; i++) {
        Message msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    // publish should work now
    Exception sendException = null;
    gotException = false;
    try {
        for (int i = 0; i < 5; i++) {
            producer.send(content);
        }
    } catch (Exception e) {
        gotException = true;
        sendException = e;
    }
    Assert.assertFalse(gotException, "unable to publish due to " + sendException);
    client.close();
}
Also used : Message(com.yahoo.pulsar.client.api.Message) ProducerConfiguration(com.yahoo.pulsar.client.api.ProducerConfiguration) BacklogQuota(com.yahoo.pulsar.common.policies.data.BacklogQuota) PersistentTopicStats(com.yahoo.pulsar.common.policies.data.PersistentTopicStats) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) PulsarClient(com.yahoo.pulsar.client.api.PulsarClient) ClientConfiguration(com.yahoo.pulsar.client.api.ClientConfiguration) Test(org.testng.annotations.Test)

Example 37 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerWithFixedBatchTime.

@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressionType) throws Exception {
    int numMsgs = 100;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchTime";
    final String subscriptionName = "time-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    Random random = new Random();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        // put a random sleep from 0 to 3 ms
        Thread.sleep(random.nextInt(4));
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    LOG.info("Sent {} messages, backlog is {} messages", numMsgs, topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog());
    assertTrue(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog() < numMsgs);
    producer.close();
}
Also used : CompletableFuture(java.util.concurrent.CompletableFuture) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) Random(java.util.Random) Message(com.yahoo.pulsar.client.api.Message) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ProducerConfiguration(com.yahoo.pulsar.client.api.ProducerConfiguration) Test(org.testng.annotations.Test)

Example 38 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerConsumer1kMessages.

@Test
public void testSimpleBatchProducerConsumer1kMessages() throws Exception {
    int numMsgs = 2000;
    int numMsgsInBatch = 4;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer1kMessages";
    final String subscriptionName = "pc1k-sub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(30000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    int sendError = 0;
    for (CompletableFuture<MessageId> sendFuture : sendFutureList) {
        if (sendFuture.isCompletedExceptionally()) {
            ++sendError;
        }
    }
    if (sendError != 0) {
        LOG.warn("[{}] Error sending {} messages", subscriptionName, sendError);
        numMsgs = numMsgs - sendError;
    }
    LOG.info("[{}] sent {} messages", subscriptionName, numMsgs);
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    // allow stats to be updated..
    Thread.sleep(5000);
    LOG.info("[{}] checking backlog stats..");
    rolloverPerIntervalStats();
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(1, TimeUnit.SECONDS);
        assertNotNull(msg);
        lastunackedMsg = msg;
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}
Also used : CompletableFuture(java.util.concurrent.CompletableFuture) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) Message(com.yahoo.pulsar.client.api.Message) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ProducerConfiguration(com.yahoo.pulsar.client.api.ProducerConfiguration) MessageId(com.yahoo.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 39 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class BatchMessageTest method testBatchAndNonBatchCumulativeAcks.

@Test
public void testBatchAndNonBatchCumulativeAcks() throws Exception {
    int numMsgs = 50;
    int numMsgsInBatch = numMsgs / 10;
    final String topicName = "persistent://prop/use/ns-abc/testBatchAndNonBatchCumulativeAcks";
    final String subscriptionName = "bnb-sub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    // create producer to publish non batch messages
    Producer noBatchProducer = pulsarClient.createProducer(topicName);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs / 2; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
        byte[] nobatchmsg = ("nobatch-" + i).getBytes();
        msg = MessageBuilder.create().setContent(nobatchmsg).build();
        sendFutureList.add(noBatchProducer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), (numMsgs / 2) / numMsgsInBatch + numMsgs / 2);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        LOG.info("[{}] got message position{} data {}", subscriptionName, msg.getMessageId(), String.valueOf(msg.getData()));
        if (i % 2 == 0) {
            lastunackedMsg = msg;
        } else {
            consumer.acknowledgeCumulative(msg);
            LOG.info("[{}] did cumulative ack on position{} ", subscriptionName, msg.getMessageId());
        }
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    assertTrue(((ConsumerImpl) consumer).isBatchingAckTrackerEmpty());
    consumer.close();
    producer.close();
    noBatchProducer.close();
}
Also used : CompletableFuture(java.util.concurrent.CompletableFuture) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) Message(com.yahoo.pulsar.client.api.Message) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ProducerConfiguration(com.yahoo.pulsar.client.api.ProducerConfiguration) Test(org.testng.annotations.Test)

Example 40 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerWithFixedBatchSize.

@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressionType) throws Exception {
    int numMsgs = 50;
    int numMsgsInBatch = numMsgs / 2;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchSize";
    final String subscriptionName = "sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    // we expect 2 messages in the backlog since we sent 50 messages with the batch size set to 25. We have set the
    // batch time high enough for it to not affect the number of messages in the batch
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        String receivedMessage = new String(msg.getData());
        String expectedMessage = "my-message-" + i;
        Assert.assertEquals(receivedMessage, expectedMessage, "Received message " + receivedMessage + " did not match the expected message " + expectedMessage);
    }
    consumer.close();
    producer.close();
}
Also used : CompletableFuture(java.util.concurrent.CompletableFuture) Consumer(com.yahoo.pulsar.client.api.Consumer) Producer(com.yahoo.pulsar.client.api.Producer) Message(com.yahoo.pulsar.client.api.Message) PersistentTopic(com.yahoo.pulsar.broker.service.persistent.PersistentTopic) ProducerConfiguration(com.yahoo.pulsar.client.api.ProducerConfiguration) Test(org.testng.annotations.Test)

Aggregations

Producer (com.yahoo.pulsar.client.api.Producer)105 Test (org.testng.annotations.Test)90 Consumer (com.yahoo.pulsar.client.api.Consumer)71 Message (com.yahoo.pulsar.client.api.Message)57 ConsumerConfiguration (com.yahoo.pulsar.client.api.ConsumerConfiguration)51 PersistentTopic (com.yahoo.pulsar.broker.service.persistent.PersistentTopic)35 ProducerConfiguration (com.yahoo.pulsar.client.api.ProducerConfiguration)32 PulsarClientException (com.yahoo.pulsar.client.api.PulsarClientException)30 PulsarClient (com.yahoo.pulsar.client.api.PulsarClient)27 CompletableFuture (java.util.concurrent.CompletableFuture)20 ClientConfiguration (com.yahoo.pulsar.client.api.ClientConfiguration)15 MessageId (com.yahoo.pulsar.client.api.MessageId)12 PulsarAdminException (com.yahoo.pulsar.client.admin.PulsarAdminException)11 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)11 PersistentSubscription (com.yahoo.pulsar.broker.service.persistent.PersistentSubscription)10 PersistentTopicStats (com.yahoo.pulsar.common.policies.data.PersistentTopicStats)10 BacklogQuota (com.yahoo.pulsar.common.policies.data.BacklogQuota)9 HashSet (java.util.HashSet)9 MockedPulsarServiceBaseTest (com.yahoo.pulsar.broker.auth.MockedPulsarServiceBaseTest)8 Field (java.lang.reflect.Field)7