use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class BacklogQuotaManagerTest method testProducerExceptionAndThenUnblock.
@Test
public void testProducerExceptionAndThenUnblock() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_exception));
final ClientConfiguration clientConf = new ClientConfiguration();
clientConf.setStatsInterval(0, TimeUnit.SECONDS);
final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
final String topic1 = "persistent://prop/usc/quotahold/exceptandunblock";
final String subName1 = "c1except";
boolean gotException = false;
Consumer consumer = client.subscribe(topic1, subName1);
ProducerConfiguration producerConfiguration = new ProducerConfiguration();
producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
byte[] content = new byte[1024];
Producer producer = client.createProducer(topic1, producerConfiguration);
for (int i = 0; i < 10; i++) {
producer.send(content);
}
Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
try {
// try to send over backlog quota and make sure it fails
producer.send(content);
producer.send(content);
Assert.fail("backlog quota did not exceed");
} catch (PulsarClientException ce) {
Assert.assertTrue(ce instanceof PulsarClientException.ProducerBlockedQuotaExceededException || ce instanceof PulsarClientException.TimeoutException, ce.getMessage());
gotException = true;
}
Assert.assertTrue(gotException, "backlog exceeded exception did not occur");
// now remove backlog and ensure that producer is unblockedrolloverStats();
PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
int backlog = (int) stats.subscriptions.get(subName1).msgBacklog;
for (int i = 0; i < backlog; i++) {
Message msg = consumer.receive();
consumer.acknowledge(msg);
}
Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
// publish should work now
Exception sendException = null;
gotException = false;
try {
for (int i = 0; i < 5; i++) {
producer.send(content);
}
} catch (Exception e) {
gotException = true;
sendException = e;
}
Assert.assertFalse(gotException, "unable to publish due to " + sendException);
client.close();
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class BatchMessageTest method testSimpleBatchProducerWithFixedBatchTime.
@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressionType) throws Exception {
int numMsgs = 100;
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchTime";
final String subscriptionName = "time-sub-1" + compressionType.toString();
Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
consumer.close();
ProducerConfiguration producerConf = new ProducerConfiguration();
producerConf.setCompressionType(compressionType);
producerConf.setBatchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
producerConf.setBatchingEnabled(true);
Producer producer = pulsarClient.createProducer(topicName, producerConf);
Random random = new Random();
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
// put a random sleep from 0 to 3 ms
Thread.sleep(random.nextInt(4));
byte[] message = ("msg-" + i).getBytes();
Message msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
rolloverPerIntervalStats();
assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
LOG.info("Sent {} messages, backlog is {} messages", numMsgs, topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog());
assertTrue(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog() < numMsgs);
producer.close();
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class BatchMessageTest method testSimpleBatchProducerConsumer1kMessages.
@Test
public void testSimpleBatchProducerConsumer1kMessages() throws Exception {
int numMsgs = 2000;
int numMsgsInBatch = 4;
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer1kMessages";
final String subscriptionName = "pc1k-sub-1";
Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
consumer.close();
ProducerConfiguration producerConf = new ProducerConfiguration();
producerConf.setBatchingMaxPublishDelay(30000, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(numMsgsInBatch);
producerConf.setBatchingEnabled(true);
Producer producer = pulsarClient.createProducer(topicName, producerConf);
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("msg-" + i).getBytes();
Message msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
int sendError = 0;
for (CompletableFuture<MessageId> sendFuture : sendFutureList) {
if (sendFuture.isCompletedExceptionally()) {
++sendError;
}
}
if (sendError != 0) {
LOG.warn("[{}] Error sending {} messages", subscriptionName, sendError);
numMsgs = numMsgs - sendError;
}
LOG.info("[{}] sent {} messages", subscriptionName, numMsgs);
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
// allow stats to be updated..
Thread.sleep(5000);
LOG.info("[{}] checking backlog stats..");
rolloverPerIntervalStats();
assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
consumer = pulsarClient.subscribe(topicName, subscriptionName);
Message lastunackedMsg = null;
for (int i = 0; i < numMsgs; i++) {
Message msg = consumer.receive(1, TimeUnit.SECONDS);
assertNotNull(msg);
lastunackedMsg = msg;
}
if (lastunackedMsg != null) {
consumer.acknowledgeCumulative(lastunackedMsg);
}
Thread.sleep(100);
assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
consumer.close();
producer.close();
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class BatchMessageTest method testBatchAndNonBatchCumulativeAcks.
@Test
public void testBatchAndNonBatchCumulativeAcks() throws Exception {
int numMsgs = 50;
int numMsgsInBatch = numMsgs / 10;
final String topicName = "persistent://prop/use/ns-abc/testBatchAndNonBatchCumulativeAcks";
final String subscriptionName = "bnb-sub-1";
Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
consumer.close();
ProducerConfiguration producerConf = new ProducerConfiguration();
producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(numMsgsInBatch);
producerConf.setBatchingEnabled(true);
Producer producer = pulsarClient.createProducer(topicName, producerConf);
// create producer to publish non batch messages
Producer noBatchProducer = pulsarClient.createProducer(topicName);
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs / 2; i++) {
byte[] message = ("msg-" + i).getBytes();
Message msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
byte[] nobatchmsg = ("nobatch-" + i).getBytes();
msg = MessageBuilder.create().setContent(nobatchmsg).build();
sendFutureList.add(noBatchProducer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
rolloverPerIntervalStats();
assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), (numMsgs / 2) / numMsgsInBatch + numMsgs / 2);
consumer = pulsarClient.subscribe(topicName, subscriptionName);
Message lastunackedMsg = null;
for (int i = 0; i < numMsgs; i++) {
Message msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
LOG.info("[{}] got message position{} data {}", subscriptionName, msg.getMessageId(), String.valueOf(msg.getData()));
if (i % 2 == 0) {
lastunackedMsg = msg;
} else {
consumer.acknowledgeCumulative(msg);
LOG.info("[{}] did cumulative ack on position{} ", subscriptionName, msg.getMessageId());
}
}
if (lastunackedMsg != null) {
consumer.acknowledgeCumulative(lastunackedMsg);
}
Thread.sleep(100);
assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
assertTrue(((ConsumerImpl) consumer).isBatchingAckTrackerEmpty());
consumer.close();
producer.close();
noBatchProducer.close();
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class BatchMessageTest method testSimpleBatchProducerWithFixedBatchSize.
@Test(dataProvider = "codec")
public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressionType) throws Exception {
int numMsgs = 50;
int numMsgsInBatch = numMsgs / 2;
final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerWithFixedBatchSize";
final String subscriptionName = "sub-1" + compressionType.toString();
Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
consumer.close();
ProducerConfiguration producerConf = new ProducerConfiguration();
producerConf.setCompressionType(compressionType);
producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(numMsgsInBatch);
producerConf.setBatchingEnabled(true);
Producer producer = pulsarClient.createProducer(topicName, producerConf);
List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
for (int i = 0; i < numMsgs; i++) {
byte[] message = ("my-message-" + i).getBytes();
Message msg = MessageBuilder.create().setContent(message).build();
sendFutureList.add(producer.sendAsync(msg));
}
FutureUtil.waitForAll(sendFutureList).get();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
rolloverPerIntervalStats();
assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
// we expect 2 messages in the backlog since we sent 50 messages with the batch size set to 25. We have set the
// batch time high enough for it to not affect the number of messages in the batch
assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
consumer = pulsarClient.subscribe(topicName, subscriptionName);
for (int i = 0; i < numMsgs; i++) {
Message msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
String receivedMessage = new String(msg.getData());
String expectedMessage = "my-message-" + i;
Assert.assertEquals(receivedMessage, expectedMessage, "Received message " + receivedMessage + " did not match the expected message " + expectedMessage);
}
consumer.close();
producer.close();
}
Aggregations