use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testBackoffAndReconnect.
@Test(dataProvider = "batch")
public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
log.info("-- Starting {} test --", methodName);
// Create consumer and producer
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Exclusive);
Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic4", "my-subscriber-name", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
if (batchMessageDelayMs != 0) {
producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(5);
producerConf.setBatchingEnabled(true);
}
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic4", producerConf);
// Produce messages
CompletableFuture<MessageId> lastFuture = null;
for (int i = 0; i < 10; i++) {
lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
log.info("Published message id: {}", msgId);
return msgId;
});
}
lastFuture.get();
Message msg = null;
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
log.info("Received: [{}]", new String(msg.getData()));
}
// Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
// the broker is up, the consumer should receive the duplicate messages.
log.info("-- Restarting broker --");
restartBroker();
msg = null;
log.info("Receiving duplicate messages..");
for (int i = 0; i < 10; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
log.info("Received: [{}]", new String(msg.getData()));
Assert.assertNotNull(msg, "Message cannot be null");
}
consumer.acknowledgeCumulative(msg);
consumer.close();
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testEnabledChecksumClient.
@Test
public void testEnabledChecksumClient() throws Exception {
log.info("-- Starting {} test --", methodName);
final int totalMsg = 10;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Exclusive);
Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
final int batchMessageDelayMs = 300;
if (batchMessageDelayMs != 0) {
producerConf.setBatchingEnabled(true);
producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(5);
}
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
for (int i = 0; i < totalMsg; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
Message msg = null;
Set<String> messageSet = Sets.newHashSet();
for (int i = 0; i < totalMsg; i++) {
msg = consumer.receive(5, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
log.debug("Received message: [{}]", receivedMessage);
String expectedMessage = "my-message-" + i;
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
}
// Acknowledge the consumption of all messages at once
consumer.acknowledgeCumulative(msg);
consumer.close();
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.
the class V1_ProducerConsumerTest method testConsumerBlockingWithUnAckedMessagesMultipleIteration.
/**
* Verify: iteration of a. message receive w/o acking b. stop receiving msg c. ack msgs d. started receiving msgs
*
* 1. Produce total X (1500) messages 2. Consumer consumes messages without acking until stop receiving from broker
* due to reaching ack-threshold (500) 3. Consumer acks messages after stop getting messages 4. Consumer again tries
* to consume messages 5. Consumer should be able to complete consuming all 1500 messages in 3 iteration (1500/500)
*
* @throws Exception
*/
@Test
public void testConsumerBlockingWithUnAckedMessagesMultipleIteration() throws Exception {
log.info("-- Starting {} test --", methodName);
int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
try {
final int unAckedMessagesBufferSize = 500;
final int receiverQueueSize = 10;
final int totalProducedMsgs = 1500;
// receiver consumes messages in iteration after acknowledging broker
final int totalReceiveIteration = totalProducedMsgs / unAckedMessagesBufferSize;
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setReceiverQueueSize(receiverQueueSize);
conf.setSubscriptionType(SubscriptionType.Shared);
Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
// (1) Produced Messages
for (int i = 0; i < totalProducedMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
int totalReceivedMessages = 0;
// (2) Receive Messages
for (int j = 0; j < totalReceiveIteration; j++) {
Message msg = null;
List<Message> messages = Lists.newArrayList();
for (int i = 0; i < totalProducedMsgs; i++) {
msg = consumer.receive(1, TimeUnit.SECONDS);
if (msg != null) {
messages.add(msg);
log.info("Received message: " + new String(msg.getData()));
} else {
break;
}
}
// client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
assertEquals(messages.size(), unAckedMessagesBufferSize);
// start acknowledging messages
messages.forEach(m -> {
try {
consumer.acknowledge(m);
} catch (PulsarClientException e) {
fail("ack failed", e);
}
});
totalReceivedMessages += messages.size();
}
// total received-messages should match to produced messages
assertEquals(totalReceivedMessages, totalProducedMsgs);
producer.close();
consumer.close();
log.info("-- Exiting {} test --", methodName);
} catch (Exception e) {
fail();
} finally {
pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
}
}
use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.
the class TopicTerminationTest method testSimpleTerminationMessageListener.
@Test(timeOut = 20000)
public void testSimpleTerminationMessageListener() throws Exception {
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
CountDownLatch latch = new CountDownLatch(1);
org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").messageListener(new MessageListener<byte[]>() {
@Override
public void received(Consumer<byte[]> consumer, Message<byte[]> msg) {
// do nothing
}
@Override
public void reachedEndOfTopic(Consumer<byte[]> consumer) {
latch.countDown();
assertTrue(consumer.hasReachedEndOfTopic());
}
}).subscribe();
/* MessageId msgId1 = */
producer.send("test-msg-1".getBytes());
/* MessageId msgId2 = */
producer.send("test-msg-2".getBytes());
MessageId msgId3 = producer.send("test-msg-3".getBytes());
consumer.acknowledgeCumulative(msgId3);
Thread.sleep(100);
assertFalse(consumer.hasReachedEndOfTopic());
MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
assertEquals(lastMessageId, msgId3);
assertTrue(latch.await(3, TimeUnit.SECONDS));
assertTrue(consumer.hasReachedEndOfTopic());
}
use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.
the class AdminApiTest method persistentTopics.
@Test(dataProvider = "topicName")
public void persistentTopics(String topicName) throws Exception {
assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
final String persistentTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
// Force to create a topic
publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 0);
assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/" + topicName));
// create consumer and subscription
URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
Consumer<byte[]> consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 10);
PersistentTopicStats topicStats = admin.persistentTopics().getStats(persistentTopicName);
assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
assertEquals(topicStats.publishers.size(), 0);
PersistentTopicInternalStats internalStats = admin.persistentTopics().getInternalStats(persistentTopicName);
assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
assertEquals(messages.size(), 3);
for (int i = 0; i < 3; i++) {
String expectedMessage = "message-" + i;
assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
}
messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 15);
assertEquals(messages.size(), 10);
for (int i = 0; i < 10; i++) {
String expectedMessage = "message-" + i;
assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
}
admin.persistentTopics().skipMessages(persistentTopicName, "my-sub", 5);
topicStats = admin.persistentTopics().getStats(persistentTopicName);
assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 5);
admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
topicStats = admin.persistentTopics().getStats(persistentTopicName);
assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
consumer.close();
client.close();
admin.persistentTopics().deleteSubscription(persistentTopicName, "my-sub");
assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList());
topicStats = admin.persistentTopics().getStats(persistentTopicName);
assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet());
assertEquals(topicStats.publishers.size(), 0);
try {
admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
} catch (NotFoundException e) {
}
admin.persistentTopics().delete(persistentTopicName);
try {
admin.persistentTopics().delete(persistentTopicName);
fail("Should have received 404");
} catch (NotFoundException e) {
}
assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
}
Aggregations