Examples with DispatchRate org.apache.pulsar.common.policies.data.DispatchRate used on opensource projects

Example 1 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class NamespacesBase method internalGetDispatchRate.

protected DispatchRate internalGetDispatchRate() {
    validateAdminAccessOnProperty(namespaceName.getProperty());
    Policies policies = getNamespacePolicies(namespaceName);
    DispatchRate dispatchRate = policies.clusterDispatchRate.get(pulsar().getConfiguration().getClusterName());
    if (dispatchRate != null) {
        return dispatchRate;
    } else {
        throw new RestException(Status.NOT_FOUND, "Dispatch-rate is not configured for cluster " + pulsar().getConfiguration().getClusterName());
    }
}

Example 2 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testClusterPolicyOverrideConfiguration.

/**
 * <pre>
 * It verifies that cluster-throttling value gets considered when namespace-policy throttling is disabled.
 *
 *  1. Update cluster-throttling-config: topic rate-limiter has cluster-config
 *  2. Update namespace-throttling-config: topic rate-limiter has namespace-config
 *  3. Disable namespace-throttling-config: topic rate-limiter has cluster-config
 *  4. Create new topic with disable namespace-config and enabled cluster-config: it takes cluster-config
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testClusterPolicyOverrideConfiguration() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName1 = "persistent://" + namespace + "/throttlingOverride1";
    final String topicName2 = "persistent://" + namespace + "/throttlingOverride2";
    final int clusterMessageRate = 100;
    int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg();
    // (1) Update message-dispatch-rate limit
    admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", Integer.toString(clusterMessageRate));
    // sleep incrementally as zk-watch notification is async and may take some time
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg() == initValue) {
            Thread.sleep(50 + (i * 10));
        }
    }
    Assert.assertNotEquals(pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg(), initValue);
    admin.namespaces().createNamespace(namespace);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName1).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName1).get();
    // (1) Update dispatch rate on cluster-config update
    Assert.assertEquals(clusterMessageRate, topic.getDispatchRateLimiter().getDispatchRateOnMsg());
    // (2) Update namespace throttling limit
    int nsMessageRate = 500;
    DispatchRate dispatchRate = new DispatchRate(nsMessageRate, 0, 1);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    for (int i = 0; i < 5; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() != nsMessageRate) {
            Thread.sleep(50 + (i * 10));
        }
    }
    Assert.assertEquals(nsMessageRate, topic.getDispatchRateLimiter().getDispatchRateOnMsg());
    // (3) Disable namespace throttling limit will force to take cluster-config
    dispatchRate = new DispatchRate(0, 0, 1);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    for (int i = 0; i < 5; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() == nsMessageRate) {
            Thread.sleep(50 + (i * 10));
        }
    }
    Assert.assertEquals(clusterMessageRate, topic.getDispatchRateLimiter().getDispatchRateOnMsg());
    // (5) Namespace throttling is disabled so, new topic should take cluster throttling limit
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).create();
    PersistentTopic topic2 = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName2).get();
    Assert.assertEquals(clusterMessageRate, topic2.getDispatchRateLimiter().getDispatchRateOnMsg());
    producer.close();
    producer2.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 3 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testMessageRateLimitingNotReceiveAllMessages.

/**
 * verify: consumer should not receive all messages due to message-rate throttling
 *
 * @param subscription
 * @throws Exception
 */
@Test(dataProvider = "subscriptionAndDispatchRateType", timeOut = 5000)
public void testMessageRateLimitingNotReceiveAllMessages(SubscriptionType subscription, DispatchRateType dispatchRateType) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    final int messageRate = 100;
    DispatchRate dispatchRate = null;
    if (DispatchRateType.messageRate.equals(dispatchRateType)) {
        dispatchRate = new DispatchRate(messageRate, -1, 360);
    } else {
        dispatchRate = new DispatchRate(-1, messageRate, 360);
    }
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0 || topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    int numMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    // consumer should not have received all publihsed message due to message-rate throttling
    Assert.assertTrue(totalReceived.get() < messageRate * 2);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 4 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testRateLimitingMultipleConsumers.

/**
 * verify message-rate on multiple consumers with shared-subscription
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testRateLimitingMultipleConsumers() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingMultipleConsumers";
    final int messageRate = 5;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    final int numProducedMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    });
    Consumer<byte[]> consumer1 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer3 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer4 = consumerBuilder.subscribe();
    Consumer<byte[]> consumer5 = consumerBuilder.subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numProducedMessages; i++) {
        final String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // it can make sure that consumer had enough time to consume message but couldn't consume due to throttling
    Thread.sleep(500);
    // consumer should not have received all published message due to message-rate throttling
    Assert.assertNotEquals(totalReceived.get(), numProducedMessages);
    consumer1.close();
    consumer2.close();
    consumer3.close();
    consumer4.close();
    consumer5.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 5 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testMessageRateDynamicallyChange.

/**
 * verifies: message-rate change gets reflected immediately into topic at runtime
 *
 * @throws Exception
 */
@Test
public void testMessageRateDynamicallyChange() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    admin.namespaces().createNamespace(namespace);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    // (1) verify message-rate is -1 initially
    Assert.assertEquals(topic.getDispatchRateLimiter().getDispatchRateOnMsg(), -1);
    // (1) change to 100
    int messageRate = 100;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    boolean isDispatchRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0) {
            isDispatchRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isDispatchRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    // (1) change to 500
    messageRate = 500;
    dispatchRate = new DispatchRate(-1, messageRate, 360);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    isDispatchRateUpdate = false;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnByte() == messageRate) {
            isDispatchRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isDispatchRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    producer.close();
}