use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testInvalidSequence.
@Test
public void testInvalidSequence() throws Exception {
log.info("-- Starting {} test --", methodName);
PulsarClient client1 = PulsarClient.create("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT);
client1.close();
ConsumerConfiguration consumerConf = new ConsumerConfiguration();
consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
try {
client1.subscribe("persistent://my-property/use/my-ns/my-topic6", "my-subscriber-name", consumerConf);
Assert.fail("Should fail");
} catch (PulsarClientException e) {
Assert.assertTrue(e instanceof PulsarClientException.AlreadyClosedException);
}
try {
client1.createProducer("persistent://my-property/use/my-ns/my-topic6");
Assert.fail("Should fail");
} catch (PulsarClientException e) {
Assert.assertTrue(e instanceof PulsarClientException.AlreadyClosedException);
}
Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic6", "my-subscriber-name", consumerConf);
try {
Message msg = MessageBuilder.create().setContent("InvalidMessage".getBytes()).build();
consumer.acknowledge(msg);
} catch (PulsarClientException.InvalidMessageException e) {
// ok
}
consumer.close();
try {
consumer.receive();
Assert.fail("Should fail");
} catch (PulsarClientException.AlreadyClosedException e) {
// ok
}
try {
consumer.unsubscribe();
Assert.fail("Should fail");
} catch (PulsarClientException.AlreadyClosedException e) {
// ok
}
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic6");
producer.close();
try {
producer.send("message".getBytes());
Assert.fail("Should fail");
} catch (PulsarClientException.AlreadyClosedException e) {
// ok
}
}
use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testActiveAndInActiveConsumerEntryCacheBehavior.
/**
* Usecase 1: Only 1 Active Subscription - 1 subscriber - Produce Messages - EntryCache should cache messages -
* EntryCache should be cleaned : Once active subscription consumes messages
*
* Usecase 2: 2 Active Subscriptions (faster and slower) and slower gets closed - 2 subscribers - Produce Messages -
* 1 faster-subscriber consumes all messages and another slower-subscriber none - EntryCache should have cached
* messages as slower-subscriber has not consumed messages yet - close slower-subscriber - EntryCache should be
* cleared
*
* @throws Exception
*/
@Test
public void testActiveAndInActiveConsumerEntryCacheBehavior() throws Exception {
log.info("-- Starting {} test --", methodName);
final long batchMessageDelayMs = 100;
final int receiverSize = 10;
final String topicName = "cache-topic";
final String sub1 = "faster-sub1";
final String sub2 = "slower-sub2";
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Shared);
conf.setReceiverQueueSize(receiverSize);
ProducerConfiguration producerConf = new ProducerConfiguration();
if (batchMessageDelayMs != 0) {
producerConf.setBatchingEnabled(true);
producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(5);
}
/**
********** usecase-1: ************
*/
// 1. Subscriber Faster subscriber
Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
final String topic = "persistent://my-property/use/my-ns/" + topicName;
Producer producer = pulsarClient.createProducer(topic, producerConf);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
cacheField.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(cacheField, cacheField.getModifiers() & ~Modifier.FINAL);
EntryCacheImpl entryCache = spy((EntryCacheImpl) cacheField.get(ledger));
cacheField.set(ledger, entryCache);
Message msg = null;
// 2. Produce messages
for (int i = 0; i < 30; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// 3. Consume messages
for (int i = 0; i < 30; i++) {
msg = subscriber1.receive(5, TimeUnit.SECONDS);
subscriber1.acknowledge(msg);
}
// Verify: EntryCache has been invalidated
verify(entryCache, atLeastOnce()).invalidateEntries(any());
// sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
// second
//
Thread.sleep(1000);
// produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
// EntryCache
producer.send("message".getBytes());
msg = subscriber1.receive(5, TimeUnit.SECONDS);
// Verify: cache has to be cleared as there is no message needs to be consumed by active subscriber
assertEquals(entryCache.getSize(), 0, 1);
/**
********** usecase-2: ************
*/
// 1.b Subscriber slower-subscriber
Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
// Produce messages
final int moreMessages = 10;
for (int i = 0; i < receiverSize + moreMessages; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// Consume messages
for (int i = 0; i < receiverSize + moreMessages; i++) {
msg = subscriber1.receive(5, TimeUnit.SECONDS);
subscriber1.acknowledge(msg);
}
// sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
// second
//
Thread.sleep(1000);
// produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
// EntryCache
producer.send("message".getBytes());
msg = subscriber1.receive(5, TimeUnit.SECONDS);
// Verify: as active-subscriber2 has not consumed messages: EntryCache must have those entries in cache
assertTrue(entryCache.getSize() != 0);
// 3.b Close subscriber2: which will trigger cache to clear the cache
subscriber2.close();
// retry strategically until broker clean up closed subscribers and invalidate all cache entries
retryStrategically((test) -> entryCache.getSize() == 0, 5, 100);
// Verify: EntryCache should be cleared
assertTrue(entryCache.getSize() == 0);
subscriber1.close();
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testShouldNotBlockConsumerIfRedeliverBeforeReceive.
@Test
public void testShouldNotBlockConsumerIfRedeliverBeforeReceive() throws Exception {
log.info("-- Starting {} test --", methodName);
int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
int totalReceiveMsg = 0;
try {
final int receiverQueueSize = 20;
final int totalProducedMsgs = 100;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setReceiverQueueSize(receiverQueueSize);
conf.setAckTimeout(1, TimeUnit.SECONDS);
conf.setSubscriptionType(SubscriptionType.Shared);
ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// (2) wait for consumer to receive messages
Thread.sleep(1000);
assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
// (3) wait for messages to expire, we should've received more
Thread.sleep(2000);
assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
for (int i = 0; i < totalProducedMsgs; i++) {
Message msg = consumer.receive(1, TimeUnit.SECONDS);
if (msg != null) {
consumer.acknowledge(msg);
totalReceiveMsg++;
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
// total received-messages should match to produced messages
assertEquals(totalProducedMsgs, totalReceiveMsg);
producer.close();
consumer.close();
log.info("-- Exiting {} test --", methodName);
} catch (Exception e) {
fail();
} finally {
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
}
}
use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testSharedSamePriorityConsumer.
/**
* <pre>
* Verifies Dispatcher dispatches messages properly with shared-subscription consumers with combination of blocked
* and unblocked consumers.
*
* 1. Dispatcher will have 5 consumers : c1, c2, c3, c4, c5.
* Out of which : c1,c2,c4,c5 will be blocked due to MaxUnackedMessages limit.
* 2. So, dispatcher should moves round-robin and make sure it delivers unblocked consumer : c3
* </pre>
*
* @throws Exception
*/
@Test(timeOut = 5000)
public void testSharedSamePriorityConsumer() throws Exception {
log.info("-- Starting {} test --", methodName);
ConsumerConfiguration conf1 = new ConsumerConfiguration();
conf1.setSubscriptionType(SubscriptionType.Shared);
final int queueSize = 5;
conf1.setReceiverQueueSize(queueSize);
int maxUnAckMsgs = pulsar.getConfiguration().getMaxConcurrentLookupRequest();
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(queueSize);
Consumer c1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
Consumer c2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
List<Future<MessageId>> futures = Lists.newArrayList();
// Asynchronously produce messages
final int totalPublishMessages = 500;
for (int i = 0; i < totalPublishMessages; i++) {
final String message = "my-message-" + i;
Future<MessageId> future = producer.sendAsync(message.getBytes());
futures.add(future);
}
log.info("Waiting for async publish to complete");
for (Future<MessageId> future : futures) {
future.get();
}
List<Message> messages = Lists.newArrayList();
// let consumer1 and consumer2 cosume messages up to the queue will be full
for (int i = 0; i < totalPublishMessages; i++) {
Message msg = c1.receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
messages.add(msg);
} else {
break;
}
}
for (int i = 0; i < totalPublishMessages; i++) {
Message msg = c2.receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
messages.add(msg);
} else {
break;
}
}
Assert.assertEquals(queueSize * 2, messages.size());
// create new consumers with the same priority
Consumer c3 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
Consumer c4 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
Consumer c5 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
for (int i = 0; i < totalPublishMessages; i++) {
Message msg = c4.receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
messages.add(msg);
} else {
break;
}
}
for (int i = 0; i < totalPublishMessages; i++) {
Message msg = c5.receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
messages.add(msg);
} else {
break;
}
}
for (int i = 0; i < totalPublishMessages; i++) {
Message msg = c3.receive(500, TimeUnit.MILLISECONDS);
if (msg != null) {
messages.add(msg);
c3.acknowledge(msg);
} else {
break;
}
}
// total messages must be consumed by all consumers
Assert.assertEquals(messages.size(), totalPublishMessages);
// Asynchronously acknowledge upto and including the last message
producer.close();
c1.close();
c2.close();
c3.close();
c4.close();
c5.close();
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnAckMsgs);
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testMutlipleSharedConsumerBlockingWithUnAckedMessages.
/**
* Verify: Consumer1 which doesn't send ack will not impact Consumer2 which sends ack for consumed message.
*
* @param batchMessageDelayMs
* @throws Exception
*/
@Test
public void testMutlipleSharedConsumerBlockingWithUnAckedMessages() throws Exception {
log.info("-- Starting {} test --", methodName);
int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
try {
final int maxUnackedMessages = 20;
final int receiverQueueSize = 10;
final int totalProducedMsgs = 100;
int totalReceiveMessages = 0;
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setReceiverQueueSize(receiverQueueSize);
conf.setSubscriptionType(SubscriptionType.Shared);
Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// (2) Consumer1: consume without ack:
// try to consume messages: but will be able to consume number of messages = maxUnackedMessages
Message msg = null;
List<Message> messages = Lists.newArrayList();
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
if (msg != null) {
messages.add(msg);
totalReceiveMessages++;
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
// client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
assertEquals(messages.size(), maxUnackedMessages);
// (3.1) Consumer2 will start consuming messages without ack: it should stop after maxUnackedMessages
messages.clear();
for (int i = 0; i < totalProducedMsgs - maxUnackedMessages; i++) {
msg = consumer2.receive(1, TimeUnit.SECONDS);
if (msg != null) {
messages.add(msg);
totalReceiveMessages++;
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
assertEquals(messages.size(), maxUnackedMessages);
// (3.2) ack for all maxUnackedMessages
messages.forEach(m -> {
try {
consumer2.acknowledge(m);
} catch (PulsarClientException e) {
fail("shouldn't have failed ", e);
}
});
// (4) Consumer2 consumer and ack: so it should consume all remaining messages
messages.clear();
for (int i = 0; i < totalProducedMsgs - (2 * maxUnackedMessages); i++) {
msg = consumer2.receive(1, TimeUnit.SECONDS);
if (msg != null) {
messages.add(msg);
totalReceiveMessages++;
consumer2.acknowledge(msg);
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
// verify total-consumer messages = total-produce messages
assertEquals(totalProducedMsgs, totalReceiveMessages);
producer.close();
consumer1.close();
consumer2.close();
log.info("-- Exiting {} test --", methodName);
} catch (Exception e) {
fail();
} finally {
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
}
}
Aggregations