Examples with BacklogQuota com.yahoo.pulsar.common.policies.data.BacklogQuota used on opensource projects

Example 11 with BacklogQuota

use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testAheadProducerOnHold.

@Test
public void testAheadProducerOnHold() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/quotahold/hold";
    final String subName1 = "c1hold";
    final int numMsgs = 10;
    Consumer consumer = client.subscribe(topic1, subName1);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
    byte[] content = new byte[1024];
    Producer producer = client.createProducer(topic1, producerConfiguration);
    for (int i = 0; i <= numMsgs; i++) {
        try {
            producer.send(content);
            LOG.info("sent [{}]", i);
        } catch (PulsarClientException.TimeoutException cte) {
            // producer close may cause a timeout on send
            LOG.info("timeout on [{}]", i);
        }
    }
    for (int i = 0; i < numMsgs; i++) {
        consumer.receive();
        LOG.info("received [{}]", i);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertEquals(stats.publishers.size(), 0, "Number of producers on topic " + topic1 + " are [" + stats.publishers.size() + "]");
    client.close();
}

Example 12 with BacklogQuota

use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testProducerException.

@Test
public void testProducerException() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_exception));
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/quotahold/except";
    final String subName1 = "c1except";
    boolean gotException = false;
    client.subscribe(topic1, subName1);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
    byte[] content = new byte[1024];
    Producer producer = client.createProducer(topic1, producerConfiguration);
    for (int i = 0; i < 10; i++) {
        producer.send(content);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    try {
        // try to send over backlog quota and make sure it fails
        producer.send(content);
        producer.send(content);
        Assert.fail("backlog quota did not exceed");
    } catch (PulsarClientException ce) {
        Assert.assertTrue(ce instanceof PulsarClientException.ProducerBlockedQuotaExceededException || ce instanceof PulsarClientException.TimeoutException, ce.getMessage());
        gotException = true;
    }
    Assert.assertTrue(gotException, "backlog exceeded exception did not occur");
    client.close();
}

Example 13 with BacklogQuota

use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testAheadProducerOnHoldTimeout.

@Test
public void testAheadProducerOnHoldTimeout() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/quotahold/holdtimeout";
    final String subName1 = "c1holdtimeout";
    boolean gotException = false;
    client.subscribe(topic1, subName1);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
    byte[] content = new byte[1024];
    Producer producer = client.createProducer(topic1, producerConfiguration);
    for (int i = 0; i < 10; i++) {
        producer.send(content);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    try {
        // try to send over backlog quota and make sure it fails
        producer.send(content);
        producer.send(content);
        Assert.fail("backlog quota did not exceed");
    } catch (PulsarClientException.TimeoutException te) {
        gotException = true;
    }
    Assert.assertTrue(gotException, "timeout did not occur");
    client.close();
}

Example 14 with BacklogQuota

use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.

the class Namespaces method checkQuotas.

private boolean checkQuotas(Policies policies, RetentionPolicies retention) {
    Map<BacklogQuota.BacklogQuotaType, BacklogQuota> backlog_quota_map = policies.backlog_quota_map;
    if (backlog_quota_map.isEmpty() || retention.getRetentionSizeInMB() == 0) {
        return true;
    }
    BacklogQuota quota = backlog_quota_map.get(BacklogQuotaType.destination_storage);
    if (quota == null) {
        quota = pulsar().getBrokerService().getBacklogQuotaManager().getDefaultQuota();
    }
    if (quota.getLimit() >= ((long) retention.getRetentionSizeInMB() * 1024 * 1024)) {
        return false;
    }
    return true;
}

Example 15 with BacklogQuota

use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.

the class BacklogQuotaManager method handleExceededBacklogQuota.

/**
     * Handle exceeded backlog by using policies set in the zookeeper for given topic
     *
     * @param persistentTopic
     *            Topic on which backlog has been exceeded
     */
public void handleExceededBacklogQuota(PersistentTopic persistentTopic) {
    DestinationName destination = DestinationName.get(persistentTopic.getName());
    String namespace = destination.getNamespace();
    String policyPath = AdminResource.path("policies", namespace);
    BacklogQuota quota = getBacklogQuota(namespace, policyPath);
    log.info("Backlog quota exceeded for topic [{}]. Applying [{}] policy", persistentTopic.getName(), quota.getPolicy());
    switch(quota.getPolicy()) {
        case consumer_backlog_eviction:
            dropBacklog(persistentTopic, quota);
            break;
        case producer_exception:
        case producer_request_hold:
            disconnectProducers(persistentTopic);
            break;
        default:
            break;
    }
}