Examples with BacklogQuota org.apache.pulsar.common.policies.data.BacklogQuota used on opensource projects

Example 11 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class ReplicatorTest method testResumptionAfterBacklogRelaxed.

/**
 * Issue #199
 *
 * It verifies that: if the remote cluster reaches backlog quota limit, replicator temporarily stops and once the
 * backlog drains it should resume replication.
 *
 * @throws Exception
 */
@Test(timeOut = 60000, enabled = true, priority = -1)
public void testResumptionAfterBacklogRelaxed() throws Exception {
    List<RetentionPolicy> policies = Lists.newArrayList();
    policies.add(RetentionPolicy.producer_exception);
    policies.add(RetentionPolicy.producer_request_hold);
    for (RetentionPolicy policy : policies) {
        // Use 1Mb quota by default
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1 * 1024 * 1024, policy));
        Thread.sleep(200);
        TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns1/%s-%d", policy, System.currentTimeMillis()));
        // Producer on r1
        MessageProducer producer1 = new MessageProducer(url1, dest);
        // Consumer on r2
        MessageConsumer consumer2 = new MessageConsumer(url2, dest);
        // Replicator for r1 -> r2
        PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
        Replicator replicator = topic.getPersistentReplicator("r2");
        // Produce 1 message in r1. This message will be replicated immediately into r2 and it will become part of
        // local backlog
        producer1.produce(1);
        Thread.sleep(500);
        // Restrict backlog quota limit to 1 byte to stop replication
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1, policy));
        Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
        assertEquals(replicator.getStats().replicationBacklog, 0);
        // Next message will not be replicated, because r2 has reached the quota
        producer1.produce(1);
        Thread.sleep(500);
        assertEquals(replicator.getStats().replicationBacklog, 1);
        // Consumer will now drain 1 message and the replication backlog will be cleared
        consumer2.receive(1);
        // Wait until the 2nd message got delivered to consumer
        consumer2.receive(1);
        int retry = 10;
        for (int i = 0; i < retry && replicator.getStats().replicationBacklog > 0; i++) {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
        assertEquals(replicator.getStats().replicationBacklog, 0);
        producer1.close();
        consumer2.close();
    }
}

Example 12 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class AdminApiTest method backlogQuotas.

@Test
public void backlogQuotas() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop-xyz/use/ns1"), Maps.newTreeMap());
    Map<BacklogQuotaType, BacklogQuota> quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/use/ns1");
    assertEquals(quotaMap.size(), 0);
    assertEquals(quotaMap.get(BacklogQuotaType.destination_storage), null);
    admin.namespaces().setBacklogQuota("prop-xyz/use/ns1", new BacklogQuota(1 * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
    quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/use/ns1");
    assertEquals(quotaMap.size(), 1);
    assertEquals(quotaMap.get(BacklogQuotaType.destination_storage), new BacklogQuota(1 * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
    admin.namespaces().removeBacklogQuota("prop-xyz/use/ns1");
    quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/use/ns1");
    assertEquals(quotaMap.size(), 0);
    assertEquals(quotaMap.get(BacklogQuotaType.destination_storage), null);
}

Example 13 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaManagerTest method testAheadProducerOnHoldTimeout.

@Test
public void testAheadProducerOnHoldTimeout() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
    final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final String topic1 = "persistent://prop/usc/quotahold/holdtimeout";
    final String subName1 = "c1holdtimeout";
    boolean gotException = false;
    client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    byte[] content = new byte[1024];
    Producer<byte[]> producer = client.newProducer().topic(topic1).sendTimeout(2, TimeUnit.SECONDS).create();
    for (int i = 0; i < 10; i++) {
        producer.send(content);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    try {
        // try to send over backlog quota and make sure it fails
        producer.send(content);
        producer.send(content);
        Assert.fail("backlog quota did not exceed");
    } catch (PulsarClientException.TimeoutException te) {
        gotException = true;
    }
    Assert.assertTrue(gotException, "timeout did not occur");
    client.close();
}

Example 14 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaManagerTest method testEvictionMulti.

@Test
public void testEvictionMulti() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(15 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic14";
    final String subName1 = "c14";
    final String subName2 = "c24";
    final int numMsgs = 10;
    final CyclicBarrier barrier = new CyclicBarrier(4);
    final CountDownLatch counter = new CountDownLatch(4);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    final Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    final PulsarClient client3 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Thread producerThread1 = new Thread() {

        public void run() {
            try {
                barrier.await();
                org.apache.pulsar.client.api.Producer<byte[]> producer = client2.newProducer().topic(topic1).create();
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread producerThread2 = new Thread() {

        public void run() {
            try {
                barrier.await();
                org.apache.pulsar.client.api.Producer<byte[]> producer = client3.newProducer().topic(topic1).create();
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread1 = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs * 2; i++) {
                    consumer1.acknowledge(consumer1.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread2 = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs * 2; i++) {
                    consumer2.acknowledge(consumer2.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread1.start();
    producerThread2.start();
    ConsumerThread1.start();
    ConsumerThread2.start();
    counter.await(20, TimeUnit.SECONDS);
    assertTrue(!gotException.get());
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize <= 15 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
    client2.close();
    client3.close();
}

Example 15 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaManagerTest method testAheadProducerOnHold.

@Test
public void testAheadProducerOnHold() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
    final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final String topic1 = "persistent://prop/usc/quotahold/hold";
    final String subName1 = "c1hold";
    final int numMsgs = 10;
    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    byte[] content = new byte[1024];
    Producer<byte[]> producer = client.newProducer().topic(topic1).sendTimeout(2, TimeUnit.SECONDS).create();
    for (int i = 0; i <= numMsgs; i++) {
        try {
            producer.send(content);
            LOG.info("sent [{}]", i);
        } catch (PulsarClientException.TimeoutException cte) {
            // producer close may cause a timeout on send
            LOG.info("timeout on [{}]", i);
        }
    }
    for (int i = 0; i < numMsgs; i++) {
        consumer.receive();
        LOG.info("received [{}]", i);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertEquals(stats.publishers.size(), 0, "Number of producers on topic " + topic1 + " are [" + stats.publishers.size() + "]");
    client.close();
}