Examples with DispatchRate org.apache.pulsar.common.policies.data.DispatchRate used on opensource projects

Example 11 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testMessageRateLimitingReceiveAllMessagesAfterThrottling.

/**
 * verify rate-limiting should throttle message-dispatching based on message-rate
 *
 * <pre>
 *  1. dispatch-msg-rate = 10 msg/sec
 *  2. send 20 msgs
 *  3. it should take up to 2 second to receive all messages
 * </pre>
 *
 * @param subscription
 * @throws Exception
 */
@Test(dataProvider = "subscriptions", timeOut = 5000)
public void testMessageRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionType subscription) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingAll";
    final int messageRate = 10;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 1);
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    final int numProducedMessages = 20;
    final CountDownLatch latch = new CountDownLatch(numProducedMessages);
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
        latch.countDown();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numProducedMessages; i++) {
        final String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    latch.await();
    Assert.assertEquals(totalReceived.get(), numProducedMessages);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 12 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testBytesRateLimitingReceiveAllMessagesAfterThrottling.

/**
 * verify rate-limiting should throttle message-dispatching based on byte-rate
 *
 * <pre>
 *  1. dispatch-byte-rate = 100 bytes/sec
 *  2. send 20 msgs : each with 10 byte
 *  3. it should take up to 2 second to receive all messages
 * </pre>
 *
 * @param subscription
 * @throws Exception
 */
@Test(dataProvider = "subscriptions", timeOut = 5000)
public void testBytesRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionType subscription) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingAll";
    final int byteRate = 100;
    DispatchRate dispatchRate = new DispatchRate(-1, byteRate, 1);
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    final int numProducedMessages = 20;
    final CountDownLatch latch = new CountDownLatch(numProducedMessages);
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
        latch.countDown();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numProducedMessages; i++) {
        producer.send(new byte[byteRate / 10]);
    }
    latch.await();
    Assert.assertEquals(totalReceived.get(), numProducedMessages);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 13 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class DispatchRateLimiter method registerLocalPoliciesListener.

/**
 * Register listener on namespace policy change to update dispatch-rate if required
 */
private void registerLocalPoliciesListener() {
    brokerService.pulsar().getConfigurationCache().policiesCache().registerListener((path, data, stat) -> {
        final NamespaceName namespace = TopicName.get(this.topicName).getNamespaceObject();
        final String cluster = brokerService.pulsar().getConfiguration().getClusterName();
        final String policiesPath = path(POLICIES, namespace.toString());
        if (policiesPath.equals(path)) {
            DispatchRate dispatchRate = data.clusterDispatchRate.get(cluster);
            // update dispatch-rate only if it's configured in policies else ignore
            if (dispatchRate != null) {
                final DispatchRate clusterDispatchRate = new DispatchRate(brokerService.pulsar().getConfiguration().getDispatchThrottlingRatePerTopicInMsg(), brokerService.pulsar().getConfiguration().getDispatchThrottlingRatePerTopicInByte(), 1);
                // cluster-throttling rate
                if (!isDispatchRateEnabled(dispatchRate) && isDispatchRateEnabled(clusterDispatchRate)) {
                    dispatchRate = clusterDispatchRate;
                }
                updateDispatchRate(dispatchRate);
            }
        }
    });
}