use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.
the class MessageDispatchThrottlingTest method testMessageRateLimitingReceiveAllMessagesAfterThrottling.
/**
* verify rate-limiting should throttle message-dispatching based on message-rate
*
* <pre>
* 1. dispatch-msg-rate = 10 msg/sec
* 2. send 20 msgs
* 3. it should take up to 2 second to receive all messages
* </pre>
*
* @param subscription
* @throws Exception
*/
@Test(dataProvider = "subscriptions", timeOut = 5000)
public void testMessageRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionType subscription) throws Exception {
log.info("-- Starting {} test --", methodName);
final String namespace = "my-property/use/throttling_ns";
final String topicName = "persistent://" + namespace + "/throttlingAll";
final int messageRate = 10;
DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 1);
admin.namespaces().createNamespace(namespace);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
for (int i = 0; i < retry; i++) {
if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0) {
isMessageRateUpdate = true;
break;
} else {
if (i != retry - 1) {
Thread.sleep(100);
}
}
}
Assert.assertTrue(isMessageRateUpdate);
Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
final int numProducedMessages = 20;
final CountDownLatch latch = new CountDownLatch(numProducedMessages);
final AtomicInteger totalReceived = new AtomicInteger(0);
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
Assert.assertNotNull(msg, "Message cannot be null");
String receivedMessage = new String(msg.getData());
log.debug("Received message [{}] in the listener", receivedMessage);
totalReceived.incrementAndGet();
latch.countDown();
}).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
// Asynchronously produce messages
for (int i = 0; i < numProducedMessages; i++) {
final String message = "my-message-" + i;
producer.send(message.getBytes());
}
latch.await();
Assert.assertEquals(totalReceived.get(), numProducedMessages);
consumer.close();
producer.close();
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.
the class MessageDispatchThrottlingTest method testBytesRateLimitingReceiveAllMessagesAfterThrottling.
/**
* verify rate-limiting should throttle message-dispatching based on byte-rate
*
* <pre>
* 1. dispatch-byte-rate = 100 bytes/sec
* 2. send 20 msgs : each with 10 byte
* 3. it should take up to 2 second to receive all messages
* </pre>
*
* @param subscription
* @throws Exception
*/
@Test(dataProvider = "subscriptions", timeOut = 5000)
public void testBytesRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionType subscription) throws Exception {
log.info("-- Starting {} test --", methodName);
final String namespace = "my-property/use/throttling_ns";
final String topicName = "persistent://" + namespace + "/throttlingAll";
final int byteRate = 100;
DispatchRate dispatchRate = new DispatchRate(-1, byteRate, 1);
admin.namespaces().createNamespace(namespace);
admin.namespaces().setDispatchRate(namespace, dispatchRate);
// create producer and topic
Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
boolean isMessageRateUpdate = false;
int retry = 5;
for (int i = 0; i < retry; i++) {
if (topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
isMessageRateUpdate = true;
break;
} else {
if (i != retry - 1) {
Thread.sleep(100);
}
}
}
Assert.assertTrue(isMessageRateUpdate);
Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
final int numProducedMessages = 20;
final CountDownLatch latch = new CountDownLatch(numProducedMessages);
final AtomicInteger totalReceived = new AtomicInteger(0);
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
Assert.assertNotNull(msg, "Message cannot be null");
String receivedMessage = new String(msg.getData());
log.debug("Received message [{}] in the listener", receivedMessage);
totalReceived.incrementAndGet();
latch.countDown();
}).subscribe();
// deactive cursors
deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
// Asynchronously produce messages
for (int i = 0; i < numProducedMessages; i++) {
producer.send(new byte[byteRate / 10]);
}
latch.await();
Assert.assertEquals(totalReceived.get(), numProducedMessages);
consumer.close();
producer.close();
log.info("-- Exiting {} test --", methodName);
}
use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.
the class DispatchRateLimiter method registerLocalPoliciesListener.
/**
* Register listener on namespace policy change to update dispatch-rate if required
*/
private void registerLocalPoliciesListener() {
brokerService.pulsar().getConfigurationCache().policiesCache().registerListener((path, data, stat) -> {
final NamespaceName namespace = TopicName.get(this.topicName).getNamespaceObject();
final String cluster = brokerService.pulsar().getConfiguration().getClusterName();
final String policiesPath = path(POLICIES, namespace.toString());
if (policiesPath.equals(path)) {
DispatchRate dispatchRate = data.clusterDispatchRate.get(cluster);
// update dispatch-rate only if it's configured in policies else ignore
if (dispatchRate != null) {
final DispatchRate clusterDispatchRate = new DispatchRate(brokerService.pulsar().getConfiguration().getDispatchThrottlingRatePerTopicInMsg(), brokerService.pulsar().getConfiguration().getDispatchThrottlingRatePerTopicInByte(), 1);
// cluster-throttling rate
if (!isDispatchRateEnabled(dispatchRate) && isDispatchRateEnabled(clusterDispatchRate)) {
dispatchRate = clusterDispatchRate;
}
updateDispatchRate(dispatchRate);
}
}
});
}
Aggregations