Examples with DispatchRate org.apache.pulsar.common.policies.data.DispatchRate used on opensource projects

Example 6 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class DispatchRateLimiter method getPoliciesDispatchRate.

/**
 * Gets configured dispatch-rate from namespace policies. Returns null if dispatch-rate is not configured
 *
 * @return
 */
public DispatchRate getPoliciesDispatchRate() {
    final NamespaceName namespace = TopicName.get(this.topicName).getNamespaceObject();
    final String cluster = brokerService.pulsar().getConfiguration().getClusterName();
    final String path = path(POLICIES, namespace.toString());
    Optional<Policies> policies = Optional.empty();
    try {
        policies = brokerService.pulsar().getConfigurationCache().policiesCache().getAsync(path).get(cacheTimeOutInSec, SECONDS);
    } catch (Exception e) {
        log.warn("Failed to get message-rate for {}", this.topicName, e);
    }
    // return policy-dispatch rate only if it's enabled in policies
    return policies.map(p -> {
        DispatchRate dispatchRate = p.clusterDispatchRate.get(cluster);
        return isDispatchRateEnabled(dispatchRate) ? dispatchRate : null;
    }).orElse(null);
}

Example 7 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class DispatchRateLimiter method updateDispatchRate.

/**
 * Update dispatch-throttling-rate. gives first priority to namespace-policy configured dispatch rate else applies
 * default broker dispatch-throttling-rate
 */
public void updateDispatchRate() {
    DispatchRate dispatchRate = getPoliciesDispatchRate();
    if (dispatchRate == null) {
        dispatchRate = new DispatchRate(brokerService.pulsar().getConfiguration().getDispatchThrottlingRatePerTopicInMsg(), brokerService.pulsar().getConfiguration().getDispatchThrottlingRatePerTopicInByte(), 1);
    }
    updateDispatchRate(dispatchRate);
    log.info("[{}] configured message-dispatch rate at broker {}", this.topicName, dispatchRate);
}

Example 8 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testGlobalNamespaceThrottling.

/**
 * <pre>
 * Verifies setting dispatch-rate on global namespace.
 * 1. It sets dispatch-rate for a local cluster into global-zk.policies
 * 2. Topic fetches dispatch-rate for the local cluster from policies
 * 3. applies dispatch rate
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testGlobalNamespaceThrottling() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/global/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    final int messageRate = 5;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
    admin.clusters().createCluster("global", new ClusterData("http://global:8080"));
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("use"));
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0 || topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    int numMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    // it can make sure that consumer had enough time to consume message but couldn't consume due to throttling
    Thread.sleep(500);
    // consumer should not have received all published message due to message-rate throttling
    Assert.assertNotEquals(totalReceived.get(), numMessages);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 9 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testMessageByteRateThrottlingCombined.

/**
 * It verifies that that dispatch-throttling considers both msg/byte rate if both of them are configured together
 *
 * @param subscription
 * @throws Exception
 */
@Test(dataProvider = "subscriptions", timeOut = 5000)
public void testMessageByteRateThrottlingCombined(SubscriptionType subscription) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingAll";
    // 5 msgs per second
    final int messageRate = 5;
    // 10 bytes per second
    final long byteRate = 10;
    DispatchRate dispatchRate = new DispatchRate(messageRate, byteRate, 360);
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0 && topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    final int numProducedMessages = 200;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    });
    Consumer<byte[]> consumer = consumerBuilder.subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    consumer.close();
    // Asynchronously produce messages
    final int dataSize = 50;
    final byte[] data = new byte[dataSize];
    for (int i = 0; i < numProducedMessages; i++) {
        producer.send(data);
    }
    consumer = consumerBuilder.subscribe();
    final int totalReceivedBytes = dataSize * totalReceived.get();
    Assert.assertNotEquals(totalReceivedBytes, byteRate * 2);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 10 with DispatchRate

use of org.apache.pulsar.common.policies.data.DispatchRate in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testNonBacklogConsumerWithThrottlingEnabled.

/**
 * It verifies that broker throttles already caught-up consumer which doesn't have backlog if the flag is enabled
 *
 * @param subscription
 * @throws Exception
 */
@Test(dataProvider = "subscriptions", timeOut = 5000)
public void testNonBacklogConsumerWithThrottlingEnabled(SubscriptionType subscription) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    final int messageRate = 10;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    admin.brokers().updateDynamicConfiguration("dispatchThrottlingOnNonBacklogConsumerEnabled", Boolean.TRUE.toString());
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isUpdated = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0) {
            isUpdated = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isUpdated);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    // enable throttling for nonBacklog consumers
    conf.setDispatchThrottlingOnNonBacklogConsumerEnabled(true);
    int numMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    // consumer should not have received all publihsed message due to message-rate throttling
    Assert.assertTrue(totalReceived.get() < messageRate * 2);
    consumer.close();
    producer.close();
    // revert default value
    this.conf.setDispatchThrottlingOnNonBacklogConsumerEnabled(false);
    log.info("-- Exiting {} test --", methodName);
}