use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause.
/**
* It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
* blocked due to unacked messsages
*
* Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking
*
* @throws Exception
*/
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() throws Exception {
log.info("-- Starting {} test --", methodName);
int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
try {
final int unAckedMessagesBufferSize = 10;
final int receiverQueueSize = 20;
final int totalProducedMsgs = 20;
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setReceiverQueueSize(receiverQueueSize);
conf.setSubscriptionType(SubscriptionType.Shared);
ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
Thread.sleep(10);
}
// (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
Message msg = null;
List<Message> messages1 = Lists.newArrayList();
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
if (msg != null) {
messages1.add(msg);
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
// client should not receive all produced messages and should be blocked due to unack-messages
assertEquals(messages1.size(), unAckedMessagesBufferSize);
Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
return (MessageIdImpl) m.getMessageId();
}).collect(Collectors.toSet());
// (3) redeliver all consumed messages
consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
Thread.sleep(1000);
Set<MessageIdImpl> messages2 = Sets.newHashSet();
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
if (msg != null) {
messages2.add((MessageIdImpl) msg.getMessageId());
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
assertEquals(messages1.size(), messages2.size());
// (4) Verify: redelivered all previous unacked-consumed messages
messages2.removeAll(redeliveryMessages);
assertEquals(messages2.size(), 0);
producer.close();
consumer.close();
log.info("-- Exiting {} test --", methodName);
} catch (Exception e) {
fail();
} finally {
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
}
}
use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testAsyncProducerAndConsumer.
@Test(timeOut = 2000)
public void testAsyncProducerAndConsumer() throws Exception {
log.info("-- Starting {} test --", methodName);
final int totalMsg = 100;
final Set<String> produceMsgs = Sets.newHashSet();
final Set<String> consumeMsgs = Sets.newHashSet();
final ProducerConfiguration producerConf = new ProducerConfiguration();
final ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Exclusive);
Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
// produce message
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
for (int i = 0; i < totalMsg; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
produceMsgs.add(message);
}
log.info(" start receiving messages :");
CountDownLatch latch = new CountDownLatch(totalMsg);
// receive messages
ExecutorService executor = Executors.newFixedThreadPool(1);
receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);
latch.await();
// verify message produced correctly
assertEquals(produceMsgs.size(), totalMsg);
// verify produced and consumed messages must be exactly same
produceMsgs.removeAll(consumeMsgs);
assertTrue(produceMsgs.isEmpty());
producer.close();
consumer.close();
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testBlockUnackConsumerAckByDifferentConsumer.
/**
* Verify: Consumer2 sends ack of Consumer1 and consumer1 should be unblock if it is blocked due to unack-messages
*
* @param batchMessageDelayMs
* @throws Exception
*/
@Test
public void testBlockUnackConsumerAckByDifferentConsumer() throws Exception {
log.info("-- Starting {} test --", methodName);
int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
try {
final int maxUnackedMessages = 20;
final int receiverQueueSize = 10;
final int totalProducedMsgs = 100;
int totalReceiveMessages = 0;
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setReceiverQueueSize(receiverQueueSize);
conf.setSubscriptionType(SubscriptionType.Shared);
Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// (2) Consumer1: consume without ack:
// try to consume messages: but will be able to consume number of messages = maxUnackedMessages
Message msg = null;
List<Message> messages = Lists.newArrayList();
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
if (msg != null) {
messages.add(msg);
totalReceiveMessages++;
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
// consumer1
assertEquals(messages.size(), maxUnackedMessages);
// (3) ack for all UnackedMessages from consumer2
messages.forEach(m -> {
try {
consumer2.acknowledge(m);
} catch (PulsarClientException e) {
fail("shouldn't have failed ", e);
}
});
// (4) consumer1 will consumer remaining msgs and consumer2 will ack those messages
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
if (msg != null) {
totalReceiveMessages++;
consumer2.acknowledge(msg);
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer2.receive(1, TimeUnit.SECONDS);
if (msg != null) {
totalReceiveMessages++;
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
// verify total-consumer messages = total-produce messages
assertEquals(totalProducedMsgs, totalReceiveMessages);
producer.close();
consumer1.close();
consumer2.close();
log.info("-- Exiting {} test --", methodName);
} catch (Exception e) {
fail();
} finally {
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
}
}
use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testUnackBlockRedeliverMessages.
@Test
public void testUnackBlockRedeliverMessages() throws Exception {
log.info("-- Starting {} test --", methodName);
int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
int totalReceiveMsg = 0;
try {
final int unAckedMessagesBufferSize = 20;
final int receiverQueueSize = 10;
final int totalProducedMsgs = 100;
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setReceiverQueueSize(receiverQueueSize);
conf.setSubscriptionType(SubscriptionType.Shared);
ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
Message msg = null;
List<Message> messages = Lists.newArrayList();
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
if (msg != null) {
messages.add(msg);
totalReceiveMsg++;
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
consumer.redeliverUnacknowledgedMessages();
Thread.sleep(1000);
int alreadyConsumedMessages = messages.size();
messages.clear();
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
if (msg != null) {
consumer.acknowledge(msg);
totalReceiveMsg++;
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
// total received-messages should match to produced messages
assertEquals(totalProducedMsgs + alreadyConsumedMessages, totalReceiveMsg);
producer.close();
consumer.close();
log.info("-- Exiting {} test --", methodName);
} catch (Exception e) {
fail();
} finally {
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
}
}
use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testSharedConsumerAckDifferentConsumer.
/**
* consume message from consumer1 and send acknowledgement from different consumer subscribed under same
* subscription-name
*
* @throws Exception
*/
@Test(timeOut = 30000)
public void testSharedConsumerAckDifferentConsumer() throws Exception {
log.info("-- Starting {} test --", methodName);
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setReceiverQueueSize(1);
conf.setSubscriptionType(SubscriptionType.Shared);
Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
Message msg = null;
Set<Message> consumerMsgSet1 = Sets.newHashSet();
Set<Message> consumerMsgSet2 = Sets.newHashSet();
for (int i = 0; i < 5; i++) {
msg = consumer1.receive();
consumerMsgSet1.add(msg);
msg = consumer2.receive();
consumerMsgSet2.add(msg);
}
consumerMsgSet1.stream().forEach(m -> {
try {
consumer2.acknowledge(m);
} catch (PulsarClientException e) {
fail();
}
});
consumerMsgSet2.stream().forEach(m -> {
try {
consumer1.acknowledge(m);
} catch (PulsarClientException e) {
fail();
}
});
consumer1.redeliverUnacknowledgedMessages();
consumer2.redeliverUnacknowledgedMessages();
try {
if (consumer1.receive(100, TimeUnit.MILLISECONDS) != null || consumer2.receive(100, TimeUnit.MILLISECONDS) != null) {
fail();
}
} finally {
consumer1.close();
consumer2.close();
}
log.info("-- Exiting {} test --", methodName);
}
Aggregations