use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class ZeroQueueSizeTest method zeroQueueSizeFailoverSubscription.
@Test()
public void zeroQueueSizeFailoverSubscription() throws PulsarClientException {
String key = "zeroQueueSizeFailoverSubscription";
// 1. Config
final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
final String subscriptionName = "my-ex-subscription-" + key;
final String messagePredicate = "my-message-" + key + "-";
// 2. Create Producer
Producer producer = pulsarClient.createProducer(topicName);
// 3. Create Consumer
ConsumerConfiguration configuration = new ConsumerConfiguration();
configuration.setReceiverQueueSize(0);
configuration.setSubscriptionType(SubscriptionType.Failover);
configuration.setConsumerName("consumer-1");
ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
configuration.setConsumerName("consumer-2");
ConsumerImpl consumer2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
// 4. Produce Messages
for (int i = 0; i < totalMessages; i++) {
String message = messagePredicate + i;
producer.send(message.getBytes());
}
// 5. Consume messages
Message message;
for (int i = 0; i < totalMessages / 2; i++) {
assertEquals(consumer1.numMessagesInQueue(), 0);
message = consumer1.receive();
assertEquals(new String(message.getData()), messagePredicate + i);
assertEquals(consumer1.numMessagesInQueue(), 0);
log.info("Consumer received : " + new String(message.getData()));
}
// 6. Trigger redelivery
consumer1.redeliverUnacknowledgedMessages();
// 7. Trigger Failover
consumer1.close();
// 8. Receive messages on failed over consumer
for (int i = 0; i < totalMessages / 2; i++) {
assertEquals(consumer2.numMessagesInQueue(), 0);
message = consumer2.receive();
assertEquals(new String(message.getData()), messagePredicate + i);
assertEquals(consumer2.numMessagesInQueue(), 0);
log.info("Consumer received : " + new String(message.getData()));
}
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class ZeroQueueSizeTest method testFailedZeroQueueSizeBatchMessage.
@Test()
public void testFailedZeroQueueSizeBatchMessage() throws PulsarClientException {
int batchMessageDelayMs = 100;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Shared);
conf.setReceiverQueueSize(0);
Consumer consumer = pulsarClient.subscribe("persistent://prop-xyz/use/ns-abc/topic1", "my-subscriber-name", conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
if (batchMessageDelayMs != 0) {
producerConf.setBatchingEnabled(true);
producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(5);
}
Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns-abc/topic1", producerConf);
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
try {
consumer.receiveAsync().handle((ok, e) -> {
if (e == null) {
// as zero receiverQueueSize doesn't support batch message, must receive exception at callback.
Assert.fail();
}
return null;
});
} finally {
consumer.close();
}
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class PartitionedProducerImpl method start.
private void start() {
AtomicReference<Throwable> createFail = new AtomicReference<Throwable>();
AtomicInteger completed = new AtomicInteger();
for (int partitionIndex = 0; partitionIndex < numPartitions; partitionIndex++) {
String partitionName = DestinationName.get(topic).getPartition(partitionIndex).toString();
ProducerImpl producer = new ProducerImpl(client, partitionName, null, conf, new CompletableFuture<Producer>(), partitionIndex);
producers.add(producer);
producer.producerCreatedFuture().handle((prod, createException) -> {
if (createException != null) {
setState(State.Failed);
createFail.compareAndSet(null, createException);
}
// created partitions
if (completed.incrementAndGet() == numPartitions) {
if (createFail.get() == null) {
setState(State.Ready);
producerCreatedFuture().complete(PartitionedProducerImpl.this);
log.info("[{}] Created partitioned producer", topic);
} else {
closeAsync().handle((ok, closeException) -> {
producerCreatedFuture().completeExceptionally(createFail.get());
client.cleanupProducer(this);
return null;
});
log.error("[{}] Could not create partitioned producer.", topic, createFail.get().getCause());
}
}
return null;
});
}
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class PartitionedProducerImpl method closeAsync.
@Override
public CompletableFuture<Void> closeAsync() {
if (getState() == State.Closing || getState() == State.Closed) {
return CompletableFuture.completedFuture(null);
}
setState(State.Closing);
AtomicReference<Throwable> closeFail = new AtomicReference<Throwable>();
AtomicInteger completed = new AtomicInteger(numPartitions);
CompletableFuture<Void> closeFuture = new CompletableFuture<>();
for (Producer producer : producers) {
if (producer != null) {
producer.closeAsync().handle((closed, ex) -> {
if (ex != null) {
closeFail.compareAndSet(null, ex);
}
if (completed.decrementAndGet() == 0) {
if (closeFail.get() == null) {
setState(State.Closed);
closeFuture.complete(null);
log.info("[{}] Closed Partitioned Producer", topic);
client.cleanupProducer(this);
} else {
setState(State.Failed);
closeFuture.completeExceptionally(closeFail.get());
log.error("[{}] Could not close Partitioned Producer", topic, closeFail.get().getCause());
}
}
return null;
});
}
}
return closeFuture;
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class BrokerClientIntegrationTest method testUnsupportedBatchMessageConsumer.
/**
* It verifies that consumer which doesn't support batch-message:
* <p>
* 1. broker disconnects that consumer
* <p>
* 2. redeliver all those messages to other supported consumer under the same subscription
*
* @param subType
* @throws Exception
*/
@Test(timeOut = 7000, dataProvider = "subType")
public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws Exception {
log.info("-- Starting {} test --", methodName);
final int batchMessageDelayMs = 1000;
final String topicName = "persistent://my-property/use/my-ns/my-topic1";
final String subscriptionName = "my-subscriber-name" + subType;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(subType);
ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, conf);
ProducerConfiguration producerConf = new ProducerConfiguration();
if (batchMessageDelayMs != 0) {
producerConf.setBatchingEnabled(true);
producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
producerConf.setBatchingMaxMessages(20);
}
Producer producer = pulsarClient.createProducer(topicName, new ProducerConfiguration());
Producer batchProducer = pulsarClient.createProducer(topicName, producerConf);
// update consumer's version to incompatible batch-message version = Version.V3
Topic topic = pulsar.getBrokerService().getTopic(topicName).get();
com.yahoo.pulsar.broker.service.Consumer brokerConsumer = topic.getSubscriptions().get(subscriptionName).getConsumers().get(0);
Field cnxField = com.yahoo.pulsar.broker.service.Consumer.class.getDeclaredField("cnx");
cnxField.setAccessible(true);
PulsarHandler cnx = (PulsarHandler) cnxField.get(brokerConsumer);
Field versionField = PulsarHandler.class.getDeclaredField("remoteEndpointProtocolVersion");
versionField.setAccessible(true);
versionField.set(cnx, 3);
// (1) send non-batch message: consumer should be able to consume
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
Set<String> messageSet = Sets.newHashSet();
Message msg = null;
for (int i = 0; i < 10; i++) {
msg = consumer1.receive(1, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
String expectedMessage = "my-message-" + i;
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
consumer1.acknowledge(msg);
}
// Also set clientCnx of the consumer to null so, it avoid reconnection so, other consumer can consume for
// verification
consumer1.setClientCnx(null);
// (2) send batch-message which should not be able to consume: as broker will disconnect the consumer
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
batchProducer.sendAsync(message.getBytes());
}
Thread.sleep(batchMessageDelayMs);
// consumer should have not received any message as it should have been disconnected
msg = consumer1.receive(2, TimeUnit.SECONDS);
assertNull(msg);
// subscrie consumer2 with supporting batch version
pulsarClient = PulsarClient.create(brokerUrl.toString());
Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
messageSet.clear();
for (int i = 0; i < 10; i++) {
msg = consumer2.receive(1, TimeUnit.SECONDS);
String receivedMessage = new String(msg.getData());
log.debug("Received message: [{}]", receivedMessage);
String expectedMessage = "my-message-" + i;
testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
consumer2.acknowledge(msg);
}
consumer2.close();
producer.close();
batchProducer.close();
log.info("-- Exiting {} test --", methodName);
}
Aggregations