Examples with BacklogQuota org.apache.pulsar.common.policies.data.BacklogQuota used on opensource projects

Example 6 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaManagerTest method testProducerExceptionAndThenUnblock.

@Test
public void testProducerExceptionAndThenUnblock() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_exception));
    final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final String topic1 = "persistent://prop/usc/quotahold/exceptandunblock";
    final String subName1 = "c1except";
    boolean gotException = false;
    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    byte[] content = new byte[1024];
    Producer<byte[]> producer = client.newProducer().topic(topic1).sendTimeout(2, TimeUnit.SECONDS).create();
    for (int i = 0; i < 10; i++) {
        producer.send(content);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    try {
        // try to send over backlog quota and make sure it fails
        producer.send(content);
        producer.send(content);
        Assert.fail("backlog quota did not exceed");
    } catch (PulsarClientException ce) {
        Assert.assertTrue(ce instanceof PulsarClientException.ProducerBlockedQuotaExceededException || ce instanceof PulsarClientException.TimeoutException, ce.getMessage());
        gotException = true;
    }
    Assert.assertTrue(gotException, "backlog exceeded exception did not occur");
    // now remove backlog and ensure that producer is unblockedrolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    int backlog = (int) stats.subscriptions.get(subName1).msgBacklog;
    for (int i = 0; i < backlog; i++) {
        Message<?> msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    // publish should work now
    Exception sendException = null;
    gotException = false;
    try {
        for (int i = 0; i < 5; i++) {
            producer.send(content);
        }
    } catch (Exception e) {
        gotException = true;
        sendException = e;
    }
    Assert.assertFalse(gotException, "unable to publish due to " + sendException);
    client.close();
}

Example 7 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaManagerTest method testConsumerBacklogEviction.

@Test
public void testConsumerBacklogEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final String topic1 = "persistent://prop/usc/ns-quota/topic1";
    final String subName1 = "c1";
    final String subName2 = "c2";
    final int numMsgs = 20;
    Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        consumer1.receive();
        consumer2.receive();
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize < 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}

Example 8 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class PersistentTopic method getBacklogQuota.

/**
 * @return Backlog quota for topic
 */
@Override
public BacklogQuota getBacklogQuota() {
    TopicName topicName = TopicName.get(this.getName());
    String namespace = topicName.getNamespace();
    String policyPath = AdminResource.path(POLICIES, namespace);
    BacklogQuota backlogQuota = brokerService.getBacklogQuotaManager().getBacklogQuota(namespace, policyPath);
    return backlogQuota;
}

Example 9 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaTest method testBacklogQuotaIdentity.

@Test
public void testBacklogQuotaIdentity() {
    Assert.assertNotEquals(new BacklogQuota(1, RetentionPolicy.producer_exception), new BacklogQuota(2, RetentionPolicy.producer_exception));
    Assert.assertNotEquals(new BacklogQuota(1, RetentionPolicy.producer_exception), new BacklogQuota(2, RetentionPolicy.consumer_backlog_eviction));
    Assert.assertNotEquals(new BacklogQuota(2, RetentionPolicy.producer_exception), new BacklogQuota(2, RetentionPolicy.consumer_backlog_eviction));
    Assert.assertEquals(new BacklogQuota(10, RetentionPolicy.producer_exception), new BacklogQuota(10, RetentionPolicy.producer_exception));
    BacklogQuota quota1 = new BacklogQuota(10, RetentionPolicy.producer_exception);
    BacklogQuota quota2 = new BacklogQuota(10, RetentionPolicy.producer_exception);
    Assert.assertEquals(quota1.hashCode(), quota2.hashCode());
}

Example 10 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class NamespacesBase method checkQuotas.

private boolean checkQuotas(Policies policies, RetentionPolicies retention) {
    Map<BacklogQuota.BacklogQuotaType, BacklogQuota> backlog_quota_map = policies.backlog_quota_map;
    if (backlog_quota_map.isEmpty() || retention.getRetentionSizeInMB() == 0 || retention.getRetentionSizeInMB() == -1) {
        return true;
    }
    BacklogQuota quota = backlog_quota_map.get(BacklogQuotaType.destination_storage);
    if (quota == null) {
        quota = pulsar().getBrokerService().getBacklogQuotaManager().getDefaultQuota();
    }
    if (quota.getLimit() >= ((long) retention.getRetentionSizeInMB() * 1024 * 1024)) {
        return false;
    }
    return true;
}