Examples with DispatchRateLimiter org.apache.pulsar.broker.service.persistent.DispatchRateLimiter used on opensource projects

Example 1 with DispatchRateLimiter

use of org.apache.pulsar.broker.service.persistent.DispatchRateLimiter in project pulsar by apache.

the class TopicPoliciesTest method testPolicyOverwrittenByNamespaceLevel.

@Test(timeOut = 20000)
public void testPolicyOverwrittenByNamespaceLevel() throws Exception {
    final String topic = testTopic + UUID.randomUUID();
    admin.topics().createNonPartitionedTopic(topic);
    DispatchRate dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(200).dispatchThrottlingRateInByte(20000).ratePeriodInSecond(1).relativeToPublishRate(true).build();
    admin.namespaces().setDispatchRate(myNamespace, dispatchRate);
    Awaitility.await().untilAsserted(() -> Assert.assertEquals(admin.namespaces().getDispatchRate(myNamespace).getDispatchThrottlingRateInMsg(), 200));
    dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(100).dispatchThrottlingRateInByte(10000).ratePeriodInSecond(1).relativeToPublishRate(true).build();
    admin.topicPolicies().setDispatchRate(topic, dispatchRate);
    Awaitility.await().untilAsserted(() -> Assert.assertNotNull(admin.topicPolicies().getDispatchRate(topic)));
    // 1 Set ns level policy, topic level should not be overwritten
    dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(300).dispatchThrottlingRateInByte(30000).ratePeriodInSecond(2).relativeToPublishRate(true).build();
    admin.namespaces().setDispatchRate(myNamespace, dispatchRate);
    Awaitility.await().untilAsserted(() -> {
        DispatchRateLimiter limiter = pulsar.getBrokerService().getTopicIfExists(topic).get().get().getDispatchRateLimiter().get();
        Assert.assertEquals(limiter.getDispatchRateOnByte(), 10000);
        Assert.assertEquals(limiter.getDispatchRateOnMsg(), 100);
    });
    admin.topicPolicies().removeDispatchRate(topic);
    Awaitility.await().untilAsserted(() -> Assert.assertNull(admin.topicPolicies().getDispatchRate(topic)));
    // 2 Remove level policy, DispatchRateLimiter should us ns level policy
    Awaitility.await().untilAsserted(() -> {
        DispatchRateLimiter limiter = pulsar.getBrokerService().getTopicIfExists(topic).get().get().getDispatchRateLimiter().get();
        Assert.assertEquals(limiter.getDispatchRateOnByte(), 30000);
        Assert.assertEquals(limiter.getDispatchRateOnMsg(), 300);
    });
}

Example 2 with DispatchRateLimiter

use of org.apache.pulsar.broker.service.persistent.DispatchRateLimiter in project pulsar by apache.

the class TopicPoliciesTest method testSubscriptionDispatchRatePolicyOverwrittenNamespaceLevel.

@Test
public void testSubscriptionDispatchRatePolicyOverwrittenNamespaceLevel() throws Exception {
    final String topic = testTopic + UUID.randomUUID();
    admin.topics().createNonPartitionedTopic(topic);
    // set namespace level subscription dispatch rate
    DispatchRate namespaceDispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(100).dispatchThrottlingRateInByte(1024 * 1024).ratePeriodInSecond(1).build();
    admin.namespaces().setSubscriptionDispatchRate(myNamespace, namespaceDispatchRate);
    Awaitility.await().untilAsserted(() -> Assert.assertEquals(admin.namespaces().getSubscriptionDispatchRate(myNamespace), namespaceDispatchRate));
    String subscriptionName = "test_subscription_rate";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().subscriptionName(subscriptionName).topic(topic).subscribe();
    // get subscription dispatch Rate limiter
    DispatchRateLimiter dispatchRateLimiter = pulsar.getBrokerService().getTopicIfExists(topic).get().get().getSubscription(subscriptionName).getDispatcher().getRateLimiter().get();
    Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnMsg(), namespaceDispatchRate.getDispatchThrottlingRateInMsg());
    Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnByte(), namespaceDispatchRate.getDispatchThrottlingRateInByte());
    // set topic level subscription dispatch rate
    DispatchRate topicDispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(200).dispatchThrottlingRateInByte(2 * 1024 * 1024).ratePeriodInSecond(1).build();
    admin.topicPolicies().setSubscriptionDispatchRate(topic, topicDispatchRate);
    Awaitility.await().untilAsserted(() -> Assert.assertEquals(admin.topicPolicies().getSubscriptionDispatchRate(topic), topicDispatchRate));
    // get subscription dispatch rate limiter
    dispatchRateLimiter = pulsar.getBrokerService().getTopicIfExists(topic).get().get().getSubscription(subscriptionName).getDispatcher().getRateLimiter().get();
    Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnByte(), topicDispatchRate.getDispatchThrottlingRateInByte());
    Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnMsg(), topicDispatchRate.getDispatchThrottlingRateInMsg());
    // remove topic level subscription dispatch rate limiter
    admin.topicPolicies().removeSubscriptionDispatchRate(topic);
    Awaitility.await().untilAsserted(() -> Assert.assertNull(admin.topicPolicies().getSubscriptionDispatchRate(topic)));
    // get subscription dispatch rate limiter
    dispatchRateLimiter = pulsar.getBrokerService().getTopicIfExists(topic).get().get().getSubscription(subscriptionName).getDispatcher().getRateLimiter().get();
    Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnByte(), namespaceDispatchRate.getDispatchThrottlingRateInByte());
    Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnMsg(), namespaceDispatchRate.getDispatchThrottlingRateInMsg());
    consumer.close();
    admin.topics().delete(topic, true);
}

Example 3 with DispatchRateLimiter

use of org.apache.pulsar.broker.service.persistent.DispatchRateLimiter in project pulsar by apache.

the class TopicPoliciesTest method testGetSetSubscriptionDispatchRate.

@Test
public void testGetSetSubscriptionDispatchRate() throws Exception {
    final String topic = testTopic + UUID.randomUUID();
    admin.topics().createNonPartitionedTopic(topic);
    DispatchRate dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(1000).dispatchThrottlingRateInByte(1024 * 1024).ratePeriodInSecond(1).build();
    log.info("Subscription Dispatch Rate: {} will set to the topic: {}", dispatchRate, topic);
    admin.topicPolicies().setSubscriptionDispatchRate(topic, dispatchRate);
    log.info("Subscription dispatch rate set success on topic: {}", topic);
    Awaitility.await().untilAsserted(() -> Assert.assertEquals(admin.topicPolicies().getSubscriptionDispatchRate(topic), dispatchRate));
    String subscriptionName = "test_subscription_rate";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().subscriptionName(subscriptionName).topic(topic).subscribe();
    DispatchRateLimiter dispatchRateLimiter = pulsar.getBrokerService().getTopicIfExists(topic).get().get().getSubscription(subscriptionName).getDispatcher().getRateLimiter().get();
    Assert.assertNotNull(dispatchRateLimiter);
    Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnByte(), dispatchRate.getDispatchThrottlingRateInByte());
    Assert.assertEquals(dispatchRateLimiter.getDispatchRateOnMsg(), dispatchRate.getDispatchThrottlingRateInMsg());
    consumer.close();
    admin.topics().delete(topic, true);
}

Example 4 with DispatchRateLimiter

use of org.apache.pulsar.broker.service.persistent.DispatchRateLimiter in project pulsar by apache.

the class SubscriptionMessageDispatchThrottlingTest method testDispatchRate.

private void testDispatchRate(SubscriptionType subscription, int brokerRate, int topicRate, int subRate, int expectRate) throws Exception {
    final String namespace = "my-property/throttling_ns";
    final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/throttlingAll");
    final String subName = "my-subscriber-name-" + subscription;
    DispatchRate subscriptionDispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(-1).dispatchThrottlingRateInByte(subRate).ratePeriodInSecond(1).build();
    DispatchRate topicDispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(-1).dispatchThrottlingRateInByte(topicRate).ratePeriodInSecond(1).build();
    admin.namespaces().createNamespace(namespace, Sets.newHashSet("test"));
    admin.namespaces().setSubscriptionDispatchRate(namespace, subscriptionDispatchRate);
    admin.namespaces().setDispatchRate(namespace, topicDispatchRate);
    admin.brokers().updateDynamicConfiguration("dispatchThrottlingRateInByte", "" + brokerRate);
    final int numProducedMessages = 30;
    final CountDownLatch latch = new CountDownLatch(numProducedMessages);
    final AtomicInteger totalReceived = new AtomicInteger(0);
    // enable throttling for nonBacklog consumers
    conf.setDispatchThrottlingOnNonBacklogConsumerEnabled(true);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(10).subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
        latch.countDown();
    }).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
    DispatchRateLimiter subRateLimiter = null;
    Dispatcher subDispatcher = topic.getSubscription(subName).getDispatcher();
    if (subDispatcher instanceof PersistentDispatcherMultipleConsumers) {
        subRateLimiter = subDispatcher.getRateLimiter().get();
    } else if (subDispatcher instanceof PersistentDispatcherSingleActiveConsumer) {
        subRateLimiter = subDispatcher.getRateLimiter().get();
    } else {
        Assert.fail("Should only have PersistentDispatcher in this test");
    }
    final DispatchRateLimiter subDispatchRateLimiter = subRateLimiter;
    Awaitility.await().atMost(Duration.ofMillis(500)).untilAsserted(() -> {
        DispatchRateLimiter brokerDispatchRateLimiter = pulsar.getBrokerService().getBrokerDispatchRateLimiter();
        Assert.assertTrue(brokerDispatchRateLimiter != null && brokerDispatchRateLimiter.getDispatchRateOnByte() > 0);
        DispatchRateLimiter topicDispatchRateLimiter = topic.getDispatchRateLimiter().orElse(null);
        Assert.assertTrue(topicDispatchRateLimiter != null && topicDispatchRateLimiter.getDispatchRateOnByte() > 0);
        Assert.assertTrue(subDispatchRateLimiter != null && subDispatchRateLimiter.getDispatchRateOnByte() > 0);
    });
    Assert.assertEquals(admin.namespaces().getSubscriptionDispatchRate(namespace).getDispatchThrottlingRateInByte(), subRate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace).getDispatchThrottlingRateInByte(), topicRate);
    long start = System.currentTimeMillis();
    // Asynchronously produce messages
    for (int i = 0; i < numProducedMessages; i++) {
        producer.send(new byte[expectRate / 10]);
    }
    latch.await();
    Assert.assertEquals(totalReceived.get(), numProducedMessages, 10);
    long end = System.currentTimeMillis();
    log.info("-- end - start: {} ", end - start);
    // first 10 messages, which equals receiverQueueSize, will not wait.
    Assert.assertTrue((end - start) >= 2500);
    Assert.assertTrue((end - start) <= 8000);
    consumer.close();
    producer.close();
    admin.topics().delete(topicName, true);
    admin.namespaces().deleteNamespace(namespace);
}

Example 5 with DispatchRateLimiter

use of org.apache.pulsar.broker.service.persistent.DispatchRateLimiter in project pulsar by apache.

the class SubscriptionMessageDispatchThrottlingTest method testClusterPolicyOverrideConfiguration.

/**
 * <pre>
 * It verifies that cluster-throttling value gets considered when namespace-policy throttling is disabled.
 *
 *  1. Update cluster-throttling-config: topic rate-limiter has cluster-config
 *  2. Update namespace-throttling-config: topic rate-limiter has namespace-config
 *  3. Disable namespace-throttling-config: topic rate-limiter has cluster-config
 *  4. Create new topic with disable namespace-config and enabled cluster-config: it takes cluster-config
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testClusterPolicyOverrideConfiguration() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/throttling_ns";
    final String topicName1 = "persistent://" + namespace + "/throttlingOverride1";
    final String topicName2 = "persistent://" + namespace + "/throttlingOverride2";
    final String subName1 = "my-subscriber-name1";
    final String subName2 = "my-subscriber-name2";
    final int clusterMessageRate = 100;
    // enable throttling for nonBacklog consumers
    conf.setDispatchThrottlingOnNonBacklogConsumerEnabled(true);
    int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerSubscriptionInMsg();
    // (1) Update message-dispatch-rate limit
    admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerSubscriptionInMsg", Integer.toString(clusterMessageRate));
    // sleep incrementally as zk-watch notification is async and may take some time
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getDispatchThrottlingRatePerSubscriptionInMsg() == initValue) {
            Thread.sleep(50 + (i * 10));
        }
    }
    Assert.assertNotEquals(pulsar.getConfiguration().getDispatchThrottlingRatePerSubscriptionInMsg(), initValue);
    admin.namespaces().createNamespace(namespace, Sets.newHashSet("test"));
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName1).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName1).get();
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName1).subscriptionName(subName1).subscribe();
    DispatchRateLimiter subRateLimiter = null;
    Dispatcher subDispatcher = topic.getSubscription(subName1).getDispatcher();
    if (subDispatcher instanceof PersistentDispatcherMultipleConsumers) {
        subRateLimiter = subDispatcher.getRateLimiter().get();
    } else if (subDispatcher instanceof PersistentDispatcherSingleActiveConsumer) {
        subRateLimiter = subDispatcher.getRateLimiter().get();
    } else {
        Assert.fail("Should only have PersistentDispatcher in this test");
    }
    // (1) Update dispatch rate on cluster-config update
    Assert.assertEquals(clusterMessageRate, subRateLimiter.getDispatchRateOnMsg());
    // (2) Update namespace throttling limit
    int nsMessageRate = 500;
    DispatchRate dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(nsMessageRate).dispatchThrottlingRateInByte(0).ratePeriodInSecond(1).build();
    admin.namespaces().setSubscriptionDispatchRate(namespace, dispatchRate);
    subRateLimiter = subDispatcher.getRateLimiter().get();
    for (int i = 0; i < 5; i++) {
        if (subRateLimiter.getDispatchRateOnMsg() != nsMessageRate) {
            Thread.sleep(50 + (i * 10));
        }
    }
    Assert.assertEquals(nsMessageRate, subRateLimiter.getDispatchRateOnMsg());
    // (3) Disable namespace throttling limit will force to take cluster-config
    dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(0).dispatchThrottlingRateInByte(0).ratePeriodInSecond(1).build();
    admin.namespaces().setSubscriptionDispatchRate(namespace, dispatchRate);
    for (int i = 0; i < 5; i++) {
        if (subRateLimiter.getDispatchRateOnMsg() == nsMessageRate) {
            Thread.sleep(50 + (i * 10));
        }
    }
    Assert.assertEquals(clusterMessageRate, subRateLimiter.getDispatchRateOnMsg());
    // (5) Namespace throttling is disabled so, new topic should take cluster throttling limit
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).create();
    PersistentTopic topic2 = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName2).get();
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName2).subscriptionName(subName2).subscribe();
    subDispatcher = topic2.getSubscription(subName2).getDispatcher();
    if (subDispatcher instanceof PersistentDispatcherMultipleConsumers) {
        subRateLimiter = subDispatcher.getRateLimiter().get();
    } else if (subDispatcher instanceof PersistentDispatcherSingleActiveConsumer) {
        subRateLimiter = subDispatcher.getRateLimiter().get();
    } else {
        Assert.fail("Should only have PersistentDispatcher in this test");
    }
    Assert.assertEquals(clusterMessageRate, subRateLimiter.getDispatchRateOnMsg());
    producer.close();
    producer2.close();
    log.info("-- Exiting {} test --", methodName);
}