use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.
the class SimpleProducerConsumerTest method testDeactivatingBacklogConsumer.
@Test
public void testDeactivatingBacklogConsumer() throws Exception {
log.info("-- Starting {} test --", methodName);
final long batchMessageDelayMs = 100;
final int receiverSize = 10;
final String topicName = "cache-topic";
final String topic = "persistent://my-property/use/my-ns/" + topicName;
final String sub1 = "faster-sub1";
final String sub2 = "slower-sub2";
// 1. Subscriber Faster subscriber: let it consume all messages immediately
Consumer<byte[]> subscriber1 = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/" + topicName).subscriptionName(sub1).subscriptionType(SubscriptionType.Shared).receiverQueueSize(receiverSize).subscribe();
// 1.b. Subscriber Slow subscriber:
Consumer<byte[]> subscriber2 = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/" + topicName).subscriptionName(sub2).subscriptionType(SubscriptionType.Shared).receiverQueueSize(receiverSize).subscribe();
ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topic);
if (batchMessageDelayMs != 0) {
producerBuilder.enableBatching(true).batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS).batchingMaxMessages(5);
}
Producer<byte[]> producer = producerBuilder.create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
// reflection to set/get cache-backlog fields value:
final long maxMessageCacheRetentionTimeMillis = 100;
Field backlogThresholdField = ManagedLedgerImpl.class.getDeclaredField("maxActiveCursorBacklogEntries");
backlogThresholdField.setAccessible(true);
Field field = ManagedLedgerImpl.class.getDeclaredField("maxMessageCacheRetentionTimeMillis");
field.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
field.set(ledger, maxMessageCacheRetentionTimeMillis);
final long maxActiveCursorBacklogEntries = (long) backlogThresholdField.get(ledger);
Message<byte[]> msg = null;
final int totalMsgs = (int) maxActiveCursorBacklogEntries + receiverSize + 1;
// 2. Produce messages
for (int i = 0; i < totalMsgs; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// 3. Consume messages: at Faster subscriber
for (int i = 0; i < totalMsgs; i++) {
msg = subscriber1.receive(100, TimeUnit.MILLISECONDS);
subscriber1.acknowledge(msg);
}
// wait : so message can be eligible to to be evict from cache
Thread.sleep(maxMessageCacheRetentionTimeMillis);
// 4. deactivate subscriber which has built the backlog
ledger.checkBackloggedCursors();
Thread.sleep(100);
// 5. verify: active subscribers
Set<String> activeSubscriber = Sets.newHashSet();
ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
assertTrue(activeSubscriber.contains(sub1));
assertFalse(activeSubscriber.contains(sub2));
// 6. consume messages : at slower subscriber
for (int i = 0; i < totalMsgs; i++) {
msg = subscriber2.receive(100, TimeUnit.MILLISECONDS);
subscriber2.acknowledge(msg);
}
ledger.checkBackloggedCursors();
activeSubscriber.clear();
ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
assertTrue(activeSubscriber.contains(sub1));
assertTrue(activeSubscriber.contains(sub2));
}
use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.
the class SimpleProducerConsumerTest method testActiveAndInActiveConsumerEntryCacheBehavior.
/**
* Usecase 1: Only 1 Active Subscription - 1 subscriber - Produce Messages - EntryCache should cache messages -
* EntryCache should be cleaned : Once active subscription consumes messages
*
* Usecase 2: 2 Active Subscriptions (faster and slower) and slower gets closed - 2 subscribers - Produce Messages -
* 1 faster-subscriber consumes all messages and another slower-subscriber none - EntryCache should have cached
* messages as slower-subscriber has not consumed messages yet - close slower-subscriber - EntryCache should be
* cleared
*
* @throws Exception
*/
@Test
public void testActiveAndInActiveConsumerEntryCacheBehavior() throws Exception {
log.info("-- Starting {} test --", methodName);
final long batchMessageDelayMs = 100;
final int receiverSize = 10;
final String topicName = "cache-topic";
final String sub1 = "faster-sub1";
final String sub2 = "slower-sub2";
/**
********** usecase-1: ************
*/
// 1. Subscriber Faster subscriber
Consumer<byte[]> subscriber1 = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/" + topicName).subscriptionName(sub1).subscriptionType(SubscriptionType.Shared).receiverQueueSize(receiverSize).subscribe();
final String topic = "persistent://my-property/use/my-ns/" + topicName;
ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topic);
if (batchMessageDelayMs != 0) {
producerBuilder.batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
producerBuilder.batchingMaxMessages(5);
producerBuilder.enableBatching(true);
}
Producer<byte[]> producer = producerBuilder.create();
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
cacheField.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(cacheField, cacheField.getModifiers() & ~Modifier.FINAL);
EntryCacheImpl entryCache = spy((EntryCacheImpl) cacheField.get(ledger));
cacheField.set(ledger, entryCache);
Message<byte[]> msg = null;
// 2. Produce messages
for (int i = 0; i < 30; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// 3. Consume messages
for (int i = 0; i < 30; i++) {
msg = subscriber1.receive(5, TimeUnit.SECONDS);
subscriber1.acknowledge(msg);
}
// Verify: EntryCache has been invalidated
verify(entryCache, atLeastOnce()).invalidateEntries(any());
// sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
// second
//
Thread.sleep(1000);
// produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
// EntryCache
producer.send("message".getBytes());
msg = subscriber1.receive(5, TimeUnit.SECONDS);
// Verify: cache has to be cleared as there is no message needs to be consumed by active subscriber
assertEquals(entryCache.getSize(), 0, 1);
/**
********** usecase-2: ************
*/
// 1.b Subscriber slower-subscriber
Consumer<byte[]> subscriber2 = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/" + topicName).subscriptionName(sub2).subscribe();
// Produce messages
final int moreMessages = 10;
for (int i = 0; i < receiverSize + moreMessages; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
// Consume messages
for (int i = 0; i < receiverSize + moreMessages; i++) {
msg = subscriber1.receive(5, TimeUnit.SECONDS);
subscriber1.acknowledge(msg);
}
// sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
// second
//
Thread.sleep(1000);
// produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
// EntryCache
producer.send("message".getBytes());
msg = subscriber1.receive(5, TimeUnit.SECONDS);
// Verify: as active-subscriber2 has not consumed messages: EntryCache must have those entries in cache
assertTrue(entryCache.getSize() != 0);
// 3.b Close subscriber2: which will trigger cache to clear the cache
subscriber2.close();
// retry strategically until broker clean up closed subscribers and invalidate all cache entries
retryStrategically((test) -> entryCache.getSize() == 0, 5, 100);
// Verify: EntryCache should be cleared
assertTrue(entryCache.getSize() == 0);
subscriber1.close();
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.
the class MessageDispatchThrottlingTest method testClusterPolicyOverrideConfiguration.
/**
* <pre>
* It verifies that cluster-throttling value gets considered when namespace-policy throttling is disabled.
*
* 1. Update cluster-throttling-config: topic rate-limiter has cluster-config
* 2. Update namespace-throttling-config: topic rate-limiter has namespace-config
* 3. Disable namespace-throttling-config: topic rate-limiter has cluster-config
* 4. Create new topic with disable namespace-config and enabled cluster-config: it takes cluster-config
*
* </pre>
*
* @throws Exception
*/
@Test
public void testClusterPolicyOverrideConfiguration() throws Exception {
log.info("-- Starting {} test --", methodName);
final String namespace = "my-property/use/throttling_ns";
final String topicName1 = "persistent://" + namespace + "/throttlingOverride1";
final String topicName2 = "persistent://" + namespace + "/throttlingOverride2";
final int clusterMessageRate = 100;
int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg();
// (1) Update message-dispatch-rate limit
admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", Integer.toString(clusterMessageRate));
// sleep incrementally as zk-watch notification is async and may take some time
for (int i = 0; i < 5; i++) {
if (pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg() == initValue) {
Thread.sleep(50 + (i * 10));
}
}
Assert.assertNotEquals(pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg(), initValue);
admin.namespaces().createNamespace(namespace);
// create producer and topic
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName1).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName1).get();
// (1) Update dispatch rate on cluster-config update
Assert.assertEquals(clusterMessageRate, topic.getDispatchRateLimiter().getDispatchRateOnMsg());
// (2) Update namespace throttling limit
int nsMessageRate = 500;
DispatchRate dispatchRate = new DispatchRate(nsMessageRate, 0, 1);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
for (int i = 0; i < 5; i++) {
if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() != nsMessageRate) {
Thread.sleep(50 + (i * 10));
}
}
Assert.assertEquals(nsMessageRate, topic.getDispatchRateLimiter().getDispatchRateOnMsg());
// (3) Disable namespace throttling limit will force to take cluster-config
dispatchRate = new DispatchRate(0, 0, 1);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
for (int i = 0; i < 5; i++) {
if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() == nsMessageRate) {
Thread.sleep(50 + (i * 10));
}
}
Assert.assertEquals(clusterMessageRate, topic.getDispatchRateLimiter().getDispatchRateOnMsg());
// (5) Namespace throttling is disabled so, new topic should take cluster throttling limit
Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).create();
PersistentTopic topic2 = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName2).get();
Assert.assertEquals(clusterMessageRate, topic2.getDispatchRateLimiter().getDispatchRateOnMsg());
producer.close();
producer2.close();
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.
the class MessageDispatchThrottlingTest method testClusterMsgByteRateLimitingClusterConfig.
/**
* It verifies that dispatch-rate throttling with cluster-configuration
*
* @param subscription
* @param dispatchRateType
* @throws Exception
*/
@Test()
public void testClusterMsgByteRateLimitingClusterConfig() throws Exception {
log.info("-- Starting {} test --", methodName);
final String namespace = "my-property/use/throttling_ns";
final String topicName = "persistent://" + namespace + "/throttlingBlock";
final int messageRate = 5;
// 1MB rate enough to let all msg to be delivered
final long byteRate = 1024 * 1024;
int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg();
// (1) Update message-dispatch-rate limit
admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", Integer.toString(messageRate));
admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInByte", Long.toString(byteRate));
// sleep incrementally as zk-watch notification is async and may take some time
for (int i = 0; i < 5; i++) {
if (pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg() == initValue) {
Thread.sleep(50 + (i * 10));
}
}
Assert.assertNotEquals(pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg(), initValue);
admin.namespaces().createNamespace(namespace);
// create producer and topic
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
int numMessages = 500;
final AtomicInteger totalReceived = new AtomicInteger(0);
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
Assert.assertNotNull(msg, "Message cannot be null");
String receivedMessage = new String(msg.getData());
log.debug("Received message [{}] in the listener", receivedMessage);
totalReceived.incrementAndGet();
}).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
// Asynchronously produce messages
for (int i = 0; i < numMessages; i++) {
final String message = "my-message-" + i;
producer.send(message.getBytes());
}
// it can make sure that consumer had enough time to consume message but couldn't consume due to throttling
Thread.sleep(500);
// consumer should not have received all published message due to message-rate throttling
Assert.assertNotEquals(totalReceived.get(), numMessages);
consumer.close();
producer.close();
pulsar.getConfiguration().setDispatchThrottlingRatePerTopicInMsg(initValue);
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.
the class MessageDispatchThrottlingTest method testMessageRateLimitingNotReceiveAllMessages.
/**
* verify: consumer should not receive all messages due to message-rate throttling
*
* @param subscription
* @throws Exception
*/
@Test(dataProvider = "subscriptionAndDispatchRateType", timeOut = 5000)
public void testMessageRateLimitingNotReceiveAllMessages(SubscriptionType subscription, DispatchRateType dispatchRateType) throws Exception {
log.info("-- Starting {} test --", methodName);
final String namespace = "my-property/use/throttling_ns";
final String topicName = "persistent://" + namespace + "/throttlingBlock";
final int messageRate = 100;
DispatchRate dispatchRate = null;
if (DispatchRateType.messageRate.equals(dispatchRateType)) {
dispatchRate = new DispatchRate(messageRate, -1, 360);
} else {
dispatchRate = new DispatchRate(-1, messageRate, 360);
}
admin.namespaces().createNamespace(namespace);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
for (int i = 0; i < retry; i++) {
if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0 || topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
isMessageRateUpdate = true;
break;
} else {
if (i != retry - 1) {
Thread.sleep(100);
}
}
}
Assert.assertTrue(isMessageRateUpdate);
Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
int numMessages = 500;
final AtomicInteger totalReceived = new AtomicInteger(0);
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
Assert.assertNotNull(msg, "Message cannot be null");
String receivedMessage = new String(msg.getData());
log.debug("Received message [{}] in the listener", receivedMessage);
totalReceived.incrementAndGet();
}).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
// Asynchronously produce messages
for (int i = 0; i < numMessages; i++) {
producer.send(new byte[80]);
}
// consumer should not have received all publihsed message due to message-rate throttling
Assert.assertTrue(totalReceived.get() < messageRate * 2);
consumer.close();
producer.close();
log.info("-- Exiting {} test --", methodName);
}
Aggregations