use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.
the class ClientErrorsTest method testLookupWithDisconnection.
@Test
public void testLookupWithDisconnection() throws Exception {
final String brokerUrl = "pulsar://127.0.0.1:" + BROKER_SERVICE_PORT;
PulsarClient client = PulsarClient.create(brokerUrl);
final AtomicInteger counter = new AtomicInteger(0);
String topic = "persistent://prop/use/ns/t1";
mockBrokerService.setHandlePartitionLookup((ctx, lookup) -> {
ctx.writeAndFlush(Commands.newPartitionMetadataResponse(0, lookup.getRequestId()));
});
mockBrokerService.setHandleLookup((ctx, lookup) -> {
if (counter.incrementAndGet() == 1) {
// piggyback unknown error to relay assertion failure
ctx.close();
return;
}
ctx.writeAndFlush(Commands.newLookupResponse(brokerUrl, null, true, LookupType.Connect, lookup.getRequestId()));
});
try {
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Exclusive);
Consumer consumer = client.subscribe(topic, "sub1", conf);
} catch (Exception e) {
if (e.getMessage().equals(ASSERTION_ERROR)) {
fail("Subscribe should not retry on persistence error");
}
assertTrue(e instanceof PulsarClientException.BrokerPersistenceException);
}
mockBrokerService.resetHandlePartitionLookup();
mockBrokerService.resetHandleLookup();
client.close();
}
use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.
the class BrokerClientIntegrationTest method testUnsupportedBatchMessageConsumer.
/**
* It verifies that consumer which doesn't support batch-message:
* <p>
* 1. broker disconnects that consumer
* <p>
* 2. redeliver all those messages to other supported consumer under the same subscription
*
* @param subType
* @throws Exception
*/
@Test(timeOut = 7000, dataProvider = "subType")
public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws Exception {
log.info("-- Starting {} test --", methodName);
final int batchMessageDelayMs = 1000;
final String topicName = "persistent://my-property/use/my-ns/my-topic1";
final String subscriptionName = "my-subscriber-name" + subType;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(subType);
ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
if (batchMessageDelayMs != 0) {
producerConf.setBatchingEnabled(true);
producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(20);
}
Producer producer = pulsarClient.createProducer(topicName, new ProducerConfiguration());
Producer batchProducer = pulsarClient.createProducer(topicName, producerConf);
// update consumer's version to incompatible batch-message version = Version.V3
Topic topic = pulsar.getBrokerService().getTopic(topicName).get();
com.yahoo.pulsar.broker.service.Consumer brokerConsumer = topic.getSubscriptions().get(subscriptionName).getConsumers().get(0);
Field cnxField = com.yahoo.pulsar.broker.service.Consumer.class.getDeclaredField("cnx");
cnxField.setAccessible(true);
PulsarHandler cnx = (PulsarHandler) cnxField.get(brokerConsumer);
Field versionField = PulsarHandler.class.getDeclaredField("remoteEndpointProtocolVersion");
versionField.setAccessible(true);
versionField.set(cnx, 3);
// (1) send non-batch message: consumer should be able to consume
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
Set<String> messageSet = Sets.newHashSet();
Message msg = null;
for (int i = 0; i < 10; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
String expectedMessage = "my-message-" + i;
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
consumer1.acknowledge(msg);
}
// Also set clientCnx of the consumer to null so, it avoid reconnection so, other consumer can consume for
// verification
consumer1.setClientCnx(null);
// (2) send batch-message which should not be able to consume: as broker will disconnect the consumer
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
batchProducer.sendAsync(message.getBytes());
}
Thread.sleep(batchMessageDelayMs);
// consumer should have not received any message as it should have been disconnected
msg = consumer1.receive(2, TimeUnit.SECONDS);
assertNull(msg);
// subscrie consumer2 with supporting batch version
pulsarClient = PulsarClient.create(brokerUrl.toString());
Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
messageSet.clear();
for (int i = 0; i < 10; i++) {
msg = consumer2.receive(1, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
log.debug("Received message: [{}]", receivedMessage);
String expectedMessage = "my-message-" + i;
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
consumer2.acknowledge(msg);
}
consumer2.close();
producer.close();
batchProducer.close();
log.info("-- Exiting {} test --", methodName);
}
use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.
the class ConsumeBaseExceptionTest method testClosedConsumer.
@Test
public void testClosedConsumer() throws PulsarClientException {
Consumer consumer = null;
consumer = pulsarClient.subscribe("persistent://prop/cluster/ns/topicName", "my-subscription");
consumer.close();
Assert.assertTrue(consumer.receiveAsync().isCompletedExceptionally());
try {
consumer.receiveAsync().exceptionally(e -> {
Assert.assertTrue(e instanceof PulsarClientException.AlreadyClosedException);
return null;
}).get();
} catch (Exception e) {
Assert.fail();
}
}
use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.
the class MessageIdTest method testChecksumVersionComptability.
/**
* Verifies: different versions of broker-deployment (one broker understands Checksum and other
* doesn't in that case remove checksum before sending to broker-2)
*
* client first produce message with checksum and then retries to send message due to connection unavailable. But this time, if
* broker doesn't understand checksum: then client should remove checksum from the message before sending to broker.
*
* 1. stop broker
* 2. client compute checksum and add into message
* 3. produce 2 messages and corrupt 1 message
* 4. start broker with lower version (which doesn't support checksum)
* 5. client reconnects to broker and due to incompatibility of version: removes checksum from message
* 6. broker doesn't do checksum validation and persist message
* 7. client receives ack
*
* @throws Exception
*/
@Test
public void testChecksumVersionComptability() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic1";
// 1. producer connect
ProducerImpl prod = (ProducerImpl) pulsarClient.createProducer(topicName);
ProducerImpl producer = spy(prod);
// return higher version compare to broker : so, it forces client-producer to remove checksum from payload
doReturn(producer.brokerChecksumSupportedVersion() + 1).when(producer).brokerChecksumSupportedVersion();
doAnswer(invocationOnMock -> prod.getState()).when(producer).getState();
doAnswer(invocationOnMock -> prod.getClientCnx()).when(producer).getClientCnx();
doAnswer(invocationOnMock -> prod.cnx()).when(producer).cnx();
Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
// Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're checksums
// would have already been computed. If we change the message content at that point, it should result in a
// checksum validation error
stopBroker();
// stop timer to auto-reconnect as let spy-Producer connect to broker manually so, spy-producer object can get
// mock-value from brokerChecksumSupportedVersion
((PulsarClientImpl) pulsarClient).timer().stop();
ClientCnx mockClientCnx = spy(new ClientCnx((PulsarClientImpl) pulsarClient));
doReturn(producer.brokerChecksumSupportedVersion() - 1).when(mockClientCnx).getRemoteEndpointProtocolVersion();
prod.setClientCnx(mockClientCnx);
Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
// corrupt the message
// new content would be 'message-3'
msg2.getData()[msg2.getData().length - 1] = '3';
prod.setClientCnx(null);
// Restart the broker to have the messages published
startBroker();
// grab broker connection with mocked producer which has higher version compare to broker
prod.grabCnx();
try {
// it should not fail: as due to unsupported version of broker: client removes checksum and broker should
// ignore the checksum validation
future1.get();
future2.get();
} catch (Exception e) {
e.printStackTrace();
fail("Broker shouldn't verify checksum for corrupted message and it shouldn't fail");
}
((ConsumerImpl) consumer).grabCnx();
// We should only receive msg1
Message msg = consumer.receive(1, TimeUnit.SECONDS);
assertEquals(new String(msg.getData()), "message-1");
msg = consumer.receive(1, TimeUnit.SECONDS);
assertEquals(new String(msg.getData()), "message-3");
}
use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.
the class MessageIdTest method testCorruptMessageRemove.
/**
* Verifies: if message is corrupted before sending to broker and if broker gives checksum error: then
* 1. Client-Producer recomputes checksum with modified data
* 2. Retry message-send again
* 3. Broker verifies checksum
* 4. client receives send-ack success
*
* @throws Exception
*/
@Test
public void testCorruptMessageRemove() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/retry-topic";
ProducerConfiguration config = new ProducerConfiguration();
config.setSendTimeout(10, TimeUnit.MINUTES);
// 1. producer connect
Producer prod = pulsarClient.createProducer(topicName, config);
ProducerImpl producer = spy((ProducerImpl) prod);
Field producerIdField = ProducerImpl.class.getDeclaredField("producerId");
producerIdField.setAccessible(true);
long producerId = (long) producerIdField.get(producer);
// registered spy ProducerImpl
producer.cnx().registerProducer(producerId, producer);
Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
// 2. Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're
// checksums
// would have already been computed. If we change the message content at that point, it should result in a
// checksum validation error
// enable checksum at producer
stopBroker();
Message msg = MessageBuilder.create().setContent("message-1".getBytes()).build();
CompletableFuture<MessageId> future = producer.sendAsync(msg);
// 3. corrupt the message
// new content would be 'message-3'
msg.getData()[msg.getData().length - 1] = '2';
// 4. Restart the broker to have the messages published
startBroker();
try {
future.get();
fail("send message should have failed with checksum excetion");
} catch (Exception e) {
if (e.getCause() instanceof PulsarClientException.ChecksumException) {
//ok (callback should get checksum exception as message was modified and corrupt)
} else {
fail("Callback should have only failed with ChecksumException", e);
}
}
// 5. Verify
// (5.1) Verify: producer's recoverChecksumError and updateChecksum invoked
verify(producer, times(1)).recoverChecksumError(any(), anyLong());
verify(producer, times(1)).verifyLocalBufferIsNotCorrupted(any());
/**
* (5.3) verify: ProducerImpl.verifyLocalBufferIsNotCorrupted() => validates if message
* is corrupt
*/
MessageImpl msg2 = (MessageImpl) MessageBuilder.create().setContent("message-1".getBytes()).build();
ByteBuf payload = msg2.getDataBuffer();
Builder metadataBuilder = ((MessageImpl) msg).getMessageBuilder();
MessageMetadata msgMetadata = metadataBuilder.setProducerName("test").setSequenceId(1).setPublishTime(10L).build();
ByteBuf cmd = Commands.newSend(producerId, 1, 1, ChecksumType.Crc32c, msgMetadata, payload);
// (a) create OpSendMsg with message-data : "message-1"
OpSendMsg op = OpSendMsg.create(((MessageImpl) msg), cmd, 1, null);
// a.verify: as message is not corrupt: no need to update checksum
assertTrue(producer.verifyLocalBufferIsNotCorrupted(op));
// (b) corrupt message
// new content would be 'message-2'
msg2.getData()[msg2.getData().length - 1] = '2';
// b. verify: as message is corrupt: update checksum
assertFalse(producer.verifyLocalBufferIsNotCorrupted(op));
assertEquals(producer.getPendingQueueSize(), 0);
// [2] test-recoverChecksumError functionality
stopBroker();
MessageImpl msg1 = (MessageImpl) MessageBuilder.create().setContent("message-1".getBytes()).build();
future = producer.sendAsync(msg1);
ClientCnx cnx = spy(new ClientCnx((PulsarClientImpl) pulsarClient) {
});
String exc = "broker is already stopped";
// when client-try to recover checksum by resending to broker: throw exception as broker is stopped
doThrow(new IllegalStateException(exc)).when(cnx).ctx();
try {
producer.recoverChecksumError(cnx, 1);
fail("it should call : resendMessages() => which should throw above mocked exception");
} catch (IllegalStateException e) {
assertEquals(exc, e.getMessage());
}
producer.close();
consumer.close();
// clean reference of mocked producer
producer = null;
}
Aggregations