use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class MessageIdTest method testCorruptMessageRemove.
/**
* Verifies: if message is corrupted before sending to broker and if broker gives checksum error: then 1.
* Client-Producer recomputes checksum with modified data 2. Retry message-send again 3. Broker verifies checksum 4.
* client receives send-ack success
*
* @throws Exception
*/
@Test
public void testCorruptMessageRemove() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/retry-topic";
// 1. producer connect
ProducerImpl<byte[]> prod = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).sendTimeout(10, TimeUnit.MINUTES).create();
ProducerImpl<byte[]> producer = spy(prod);
Field producerIdField = ProducerImpl.class.getDeclaredField("producerId");
producerIdField.setAccessible(true);
long producerId = (long) producerIdField.get(producer);
// registered spy ProducerImpl
producer.cnx().registerProducer(producerId, producer);
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
// 2. Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're
// checksums
// would have already been computed. If we change the message content at that point, it should result in a
// checksum validation error
// enable checksum at producer
stopBroker();
Message<byte[]> msg = MessageBuilder.create().setContent("message-1".getBytes()).build();
CompletableFuture<MessageId> future = producer.sendAsync(msg);
// 3. corrupt the message
// new content would be 'message-3'
msg.getData()[msg.getData().length - 1] = '2';
// 4. Restart the broker to have the messages published
startBroker();
try {
future.get();
fail("send message should have failed with checksum excetion");
} catch (Exception e) {
if (e.getCause() instanceof PulsarClientException.ChecksumException) {
// ok (callback should get checksum exception as message was modified and corrupt)
} else {
fail("Callback should have only failed with ChecksumException", e);
}
}
// 5. Verify
/**
* verify: ProducerImpl.verifyLocalBufferIsNotCorrupted() => validates if message is corrupt
*/
MessageImpl<byte[]> msg2 = (MessageImpl<byte[]>) MessageBuilder.create().setContent("message-1".getBytes()).build();
ByteBuf payload = msg2.getDataBuffer();
Builder metadataBuilder = ((MessageImpl<byte[]>) msg).getMessageBuilder();
MessageMetadata msgMetadata = metadataBuilder.setProducerName("test").setSequenceId(1).setPublishTime(10L).build();
ByteBufPair cmd = Commands.newSend(producerId, 1, 1, ChecksumType.Crc32c, msgMetadata, payload);
// (a) create OpSendMsg with message-data : "message-1"
OpSendMsg op = OpSendMsg.create(((MessageImpl<byte[]>) msg), cmd, 1, null);
// a.verify: as message is not corrupt: no need to update checksum
assertTrue(producer.verifyLocalBufferIsNotCorrupted(op));
// (b) corrupt message
// new content would be 'message-2'
msg2.getData()[msg2.getData().length - 1] = '2';
// b. verify: as message is corrupt: update checksum
assertFalse(producer.verifyLocalBufferIsNotCorrupted(op));
assertEquals(producer.getPendingQueueSize(), 0);
// [2] test-recoverChecksumError functionality
stopBroker();
MessageImpl<byte[]> msg1 = (MessageImpl<byte[]>) MessageBuilder.create().setContent("message-1".getBytes()).build();
future = producer.sendAsync(msg1);
ClientCnx cnx = spy(new ClientCnx(new ClientConfigurationData(), ((PulsarClientImpl) pulsarClient).eventLoopGroup()));
String exc = "broker is already stopped";
// when client-try to recover checksum by resending to broker: throw exception as broker is stopped
doThrow(new IllegalStateException(exc)).when(cnx).ctx();
try {
producer.recoverChecksumError(cnx, 1);
fail("it should call : resendMessages() => which should throw above mocked exception");
} catch (IllegalStateException e) {
assertEquals(exc, e.getMessage());
}
producer.close();
consumer.close();
// clean reference of mocked producer
producer = null;
}
use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class MessageIdTest method producerSendAsync.
@Test(timeOut = 10000)
public void producerSendAsync() throws PulsarClientException {
// 1. Basic Config
String key = "producerSendAsync";
final String topicName = "persistent://prop/cluster/namespace/topic-" + key;
final String subscriptionName = "my-subscription-" + key;
final String messagePredicate = "my-message-" + key + "-";
final int numberOfMessages = 30;
// 2. Create Producer
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
// 3. Create Consumer
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
// 4. Publish message and get message id
Set<MessageId> messageIds = new HashSet<>();
List<Future<MessageId>> futures = new ArrayList<>();
for (int i = 0; i < numberOfMessages; i++) {
String message = messagePredicate + i;
futures.add(producer.sendAsync(message.getBytes()));
}
MessageIdImpl previousMessageId = null;
for (Future<MessageId> f : futures) {
try {
MessageIdImpl currentMessageId = (MessageIdImpl) f.get();
if (previousMessageId != null) {
Assert.assertTrue(currentMessageId.compareTo(previousMessageId) > 0, "Message Ids should be in ascending order");
}
messageIds.add(currentMessageId);
previousMessageId = currentMessageId;
} catch (Exception e) {
Assert.fail("Failed to publish message, Exception: " + e.getMessage());
}
}
// 4. Check if message Ids are correct
log.info("Message IDs = " + messageIds);
Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
for (int i = 0; i < numberOfMessages; i++) {
Message<byte[]> message = consumer.receive();
Assert.assertEquals(new String(message.getData()), messagePredicate + i);
MessageId messageId = message.getMessageId();
Assert.assertTrue(messageIds.remove(messageId), "Failed to receive message");
}
log.info("Message IDs = " + messageIds);
Assert.assertEquals(messageIds.size(), 0, "Not all messages received successfully");
consumer.unsubscribe();
}
use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class TopicsConsumerImplTest method testAsyncConsumer.
@Test(timeOut = testTimeout)
public void testAsyncConsumer() throws Exception {
String key = "TopicsConsumerAsyncTest";
final String subscriptionName = "my-ex-subscription-" + key;
final String messagePredicate = "my-message-" + key + "-";
final int totalMessages = 30;
final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
admin.properties().createProperty("prop", new PropertyAdmin());
admin.persistentTopics().createPartitionedTopic(topicName2, 2);
admin.persistentTopics().createPartitionedTopic(topicName3, 3);
// 1. producer connect
Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1).create();
Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
// 2. Create consumer
Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
assertTrue(consumer instanceof TopicsConsumerImpl);
// Asynchronously produce messages
List<Future<MessageId>> futures = Lists.newArrayList();
for (int i = 0; i < totalMessages / 3; i++) {
futures.add(producer1.sendAsync((messagePredicate + "producer1-" + i).getBytes()));
futures.add(producer2.sendAsync((messagePredicate + "producer2-" + i).getBytes()));
futures.add(producer3.sendAsync((messagePredicate + "producer3-" + i).getBytes()));
}
log.info("Waiting for async publish to complete : {}", futures.size());
for (Future<MessageId> future : futures) {
future.get();
}
log.info("start async consume");
CountDownLatch latch = new CountDownLatch(totalMessages);
ExecutorService executor = Executors.newFixedThreadPool(1);
executor.execute(() -> IntStream.range(0, totalMessages).forEach(index -> consumer.receiveAsync().thenAccept(msg -> {
assertTrue(msg instanceof TopicMessageImpl);
try {
consumer.acknowledge(msg);
} catch (PulsarClientException e1) {
fail("message acknowledge failed", e1);
}
latch.countDown();
log.info("receive index: {}, latch countDown: {}", index, latch.getCount());
}).exceptionally(ex -> {
log.warn("receive index: {}, failed receive message {}", index, ex.getMessage());
ex.printStackTrace();
return null;
})));
latch.await();
log.info("success latch wait");
consumer.unsubscribe();
consumer.close();
producer1.close();
producer2.close();
producer3.close();
}
use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class UnAcknowledgedMessagesTimeoutTest method testSharedSingleAckedPartitionedTopic.
@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
String key = "testSharedSingleAckedPartitionedTopic";
final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
final String subscriptionName = "my-shared-subscription-" + key;
final String messagePredicate = "my-message-" + key + "-";
final int totalMessages = 20;
final int numberOfPartitions = 3;
admin.properties().createProperty("prop", new PropertyAdmin());
admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
// Special step to create partitioned topic
// 1. producer connect
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
// 2. Create consumer
Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(100).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).consumerName("Consumer-1").subscribe();
Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(100).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).consumerName("Consumer-2").subscribe();
// 3. producer publish messages
for (int i = 0; i < totalMessages; i++) {
String message = messagePredicate + i;
MessageId msgId = producer.send(message.getBytes());
log.info("Message produced: {} -- msgId: {}", message, msgId);
}
// 4. Receive messages
int messageCount1 = receiveAllMessage(consumer1, false);
int messageCount2 = receiveAllMessage(consumer2, true);
int ackCount1 = 0;
int ackCount2 = messageCount2;
log.info(key + " messageCount1 = " + messageCount1);
log.info(key + " messageCount2 = " + messageCount2);
log.info(key + " ackCount1 = " + ackCount1);
log.info(key + " ackCount2 = " + ackCount2);
assertEquals(messageCount1 + messageCount2, totalMessages);
// 5. Check if Messages redelivered again
// Since receive is a blocking call hoping that timeout will kick in
Thread.sleep((int) (ackTimeOutMillis * 1.1));
log.info(key + " Timeout should be triggered now");
messageCount1 = receiveAllMessage(consumer1, true);
messageCount2 += receiveAllMessage(consumer2, false);
ackCount1 = messageCount1;
log.info(key + " messageCount1 = " + messageCount1);
log.info(key + " messageCount2 = " + messageCount2);
log.info(key + " ackCount1 = " + ackCount1);
log.info(key + " ackCount2 = " + ackCount2);
assertEquals(messageCount1 + messageCount2, totalMessages);
assertEquals(ackCount1 + messageCount2, totalMessages);
Thread.sleep((int) (ackTimeOutMillis * 1.1));
// Since receive is a blocking call hoping that timeout will kick in
log.info(key + " Timeout should be triggered again");
ackCount1 += receiveAllMessage(consumer1, true);
ackCount2 += receiveAllMessage(consumer2, true);
log.info(key + " ackCount1 = " + ackCount1);
log.info(key + " ackCount2 = " + ackCount2);
assertEquals(ackCount1 + ackCount2, totalMessages);
}
use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.
the class PersistentTopicsImpl method terminateTopicAsync.
@Override
public CompletableFuture<MessageId> terminateTopicAsync(String topic) {
TopicName tn = validateTopic(topic);
final CompletableFuture<MessageId> future = new CompletableFuture<>();
try {
final WebTarget path = topicPath(tn, "terminate");
request(path).async().post(Entity.entity("", MediaType.APPLICATION_JSON), new InvocationCallback<MessageIdImpl>() {
@Override
public void completed(MessageIdImpl messageId) {
future.complete(messageId);
}
@Override
public void failed(Throwable throwable) {
log.warn("[{}] Failed to perform http post request: {}", path.getUri(), throwable.getMessage());
future.completeExceptionally(getApiException(throwable.getCause()));
}
});
} catch (PulsarAdminException cae) {
future.completeExceptionally(cae);
}
return future;
}
Aggregations