Examples with BacklogQuota org.apache.pulsar.common.policies.data.BacklogQuota used on opensource projects

Example 1 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class PulsarAdminToolTest method namespaces.

@Test
void namespaces() throws Exception {
    PulsarAdmin admin = Mockito.mock(PulsarAdmin.class);
    Namespaces mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    Lookup mockLookup = mock(Lookup.class);
    when(admin.lookups()).thenReturn(mockLookup);
    CmdNamespaces namespaces = new CmdNamespaces(admin);
    namespaces.run(split("list myprop"));
    verify(mockNamespaces).getNamespaces("myprop");
    namespaces.run(split("list-cluster myprop/clust"));
    verify(mockNamespaces).getNamespaces("myprop", "clust");
    namespaces.run(split("topics myprop/clust/ns1"));
    verify(mockNamespaces).getTopics("myprop/clust/ns1");
    namespaces.run(split("policies myprop/clust/ns1"));
    verify(mockNamespaces).getPolicies("myprop/clust/ns1");
    namespaces.run(split("create myprop/clust/ns1"));
    verify(mockNamespaces).createNamespace("myprop/clust/ns1");
    namespaces.run(split("delete myprop/clust/ns1"));
    verify(mockNamespaces).deleteNamespace("myprop/clust/ns1");
    namespaces.run(split("permissions myprop/clust/ns1"));
    verify(mockNamespaces).getPermissions("myprop/clust/ns1");
    namespaces.run(split("grant-permission myprop/clust/ns1 --role role1 --actions produce,consume"));
    verify(mockNamespaces).grantPermissionOnNamespace("myprop/clust/ns1", "role1", EnumSet.of(AuthAction.produce, AuthAction.consume));
    namespaces.run(split("revoke-permission myprop/clust/ns1 --role role1"));
    verify(mockNamespaces).revokePermissionsOnNamespace("myprop/clust/ns1", "role1");
    namespaces.run(split("set-clusters myprop/clust/ns1 -c use,usw,usc"));
    verify(mockNamespaces).setNamespaceReplicationClusters("myprop/clust/ns1", Lists.newArrayList("use", "usw", "usc"));
    namespaces.run(split("get-clusters myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceReplicationClusters("myprop/clust/ns1");
    namespaces.run(split("unload myprop/clust/ns1"));
    verify(mockNamespaces).unload("myprop/clust/ns1");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff"));
    verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff");
    namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff"));
    verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff", false);
    namespaces.run(split("get-backlog-quotas myprop/clust/ns1"));
    verify(mockNamespaces).getBacklogQuotaMap("myprop/clust/ns1");
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_request_hold -l 10"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10, RetentionPolicy.producer_request_hold));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10K"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024, RetentionPolicy.producer_exception));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10M"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024 * 1024, RetentionPolicy.producer_exception));
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10G"));
    verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10l * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
    namespaces.run(split("set-persistence myprop/clust/ns1 -e 2 -w 1 -a 1 -r 100.0"));
    verify(mockNamespaces).setPersistence("myprop/clust/ns1", new PersistencePolicies(2, 1, 1, 100.0d));
    namespaces.run(split("get-persistence myprop/clust/ns1"));
    verify(mockNamespaces).getPersistence("myprop/clust/ns1");
    namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 300"));
    verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 300);
    namespaces.run(split("set-deduplication myprop/clust/ns1 --enable"));
    verify(mockNamespaces).setDeduplicationStatus("myprop/clust/ns1", true);
    namespaces.run(split("get-message-ttl myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceMessageTTL("myprop/clust/ns1");
    namespaces.run(split("set-anti-affinity-group myprop/clust/ns1 -g group"));
    verify(mockNamespaces).setNamespaceAntiAffinityGroup("myprop/clust/ns1", "group");
    namespaces.run(split("get-anti-affinity-group myprop/clust/ns1"));
    verify(mockNamespaces).getNamespaceAntiAffinityGroup("myprop/clust/ns1");
    namespaces.run(split("get-anti-affinity-namespaces -p dummy -c cluster -g group"));
    verify(mockNamespaces).getAntiAffinityNamespaces("dummy", "cluster", "group");
    namespaces.run(split("delete-anti-affinity-group myprop/clust/ns1 "));
    verify(mockNamespaces).deleteNamespaceAntiAffinityGroup("myprop/clust/ns1");
    namespaces.run(split("set-retention myprop/clust/ns1 -t 1h -s 1M"));
    verify(mockNamespaces).setRetention("myprop/clust/ns1", new RetentionPolicies(60, 1));
    namespaces.run(split("get-retention myprop/clust/ns1"));
    verify(mockNamespaces).getRetention("myprop/clust/ns1");
    namespaces.run(split("clear-backlog myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBacklog("myprop/clust/ns1");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBundleBacklog("myprop/clust/ns1", "0x80000000_0xffffffff");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -s my-sub myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBacklogForSubscription("myprop/clust/ns1", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1 -force"));
    verify(mockNamespaces).clearNamespaceBundleBacklogForSubscription("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
    namespaces.run(split("unsubscribe -s my-sub myprop/clust/ns1"));
    verify(mockNamespaces).unsubscribeNamespace("myprop/clust/ns1", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("unsubscribe -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1"));
    verify(mockNamespaces).unsubscribeNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
    mockNamespaces = mock(Namespaces.class);
    when(admin.namespaces()).thenReturn(mockNamespaces);
    namespaces = new CmdNamespaces(admin);
    namespaces.run(split("get-max-producers-per-topic myprop/clust/ns1"));
    verify(mockNamespaces).getMaxProducersPerTopic("myprop/clust/ns1");
    namespaces.run(split("set-max-producers-per-topic myprop/clust/ns1 -p 1"));
    verify(mockNamespaces).setMaxProducersPerTopic("myprop/clust/ns1", 1);
    namespaces.run(split("get-max-consumers-per-topic myprop/clust/ns1"));
    verify(mockNamespaces).getMaxConsumersPerTopic("myprop/clust/ns1");
    namespaces.run(split("set-max-consumers-per-topic myprop/clust/ns1 -c 2"));
    verify(mockNamespaces).setMaxConsumersPerTopic("myprop/clust/ns1", 2);
    namespaces.run(split("get-max-consumers-per-subscription myprop/clust/ns1"));
    verify(mockNamespaces).getMaxConsumersPerSubscription("myprop/clust/ns1");
    namespaces.run(split("set-max-consumers-per-subscription myprop/clust/ns1 -c 3"));
    verify(mockNamespaces).setMaxConsumersPerSubscription("myprop/clust/ns1", 3);
}

Example 2 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class ProxyPublishConsumeTest method producerBacklogQuotaExceededTest.

@Test(timeOut = 30000)
public void producerBacklogQuotaExceededTest() throws Exception {
    admin.namespaces().createNamespace("my-property/use/ns-ws-quota");
    admin.namespaces().setBacklogQuota("my-property/use/ns-ws-quota", new BacklogQuota(10, BacklogQuota.RetentionPolicy.producer_request_hold));
    final String topic = "my-property/use/ns-ws-quota/my-topic5";
    final String subscription = "my-sub";
    final String consumerUri = "ws://localhost:" + port + "/ws/consumer/persistent/" + topic + "/" + subscription;
    final String producerUri = "ws://localhost:" + port + "/ws/producer/persistent/" + topic;
    URI consumeUri = URI.create(consumerUri);
    URI produceUri = URI.create(producerUri);
    WebSocketClient consumeClient = new WebSocketClient();
    WebSocketClient produceClient1 = new WebSocketClient();
    WebSocketClient produceClient2 = new WebSocketClient();
    SimpleConsumerSocket consumeSocket = new SimpleConsumerSocket();
    SimpleProducerSocket produceSocket1 = new SimpleProducerSocket();
    SimpleProducerSocket produceSocket2 = new SimpleProducerSocket();
    // Create subscription
    try {
        consumeClient.start();
        ClientUpgradeRequest consumeRequest = new ClientUpgradeRequest();
        Future<Session> consumerFuture = consumeClient.connect(consumeSocket, consumeUri, consumeRequest);
        consumerFuture.get();
    } finally {
        stopWebSocketClient(consumeClient);
    }
    // Fill the backlog
    try {
        produceClient1.start();
        ClientUpgradeRequest produceRequest = new ClientUpgradeRequest();
        Future<Session> producerFuture = produceClient1.connect(produceSocket1, produceUri, produceRequest);
        producerFuture.get();
        produceSocket1.sendMessage(100);
    } finally {
        stopWebSocketClient(produceClient1);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    // New producer fails to connect
    try {
        produceClient2.start();
        ClientUpgradeRequest produceRequest = new ClientUpgradeRequest();
        Future<Session> producerFuture = produceClient2.connect(produceSocket2, produceUri, produceRequest);
        producerFuture.get();
        Assert.fail("should fail: backlog quota exceeded");
    } catch (Exception e) {
        // Expected
        Assert.assertTrue(e.getCause() instanceof UpgradeException);
        Assert.assertEquals(((UpgradeException) e.getCause()).getResponseStatusCode(), HttpServletResponse.SC_SERVICE_UNAVAILABLE);
    } finally {
        stopWebSocketClient(produceClient2);
        admin.persistentTopics().skipAllMessages("persistent://" + topic, subscription);
        admin.persistentTopics().delete("persistent://" + topic);
        admin.namespaces().removeBacklogQuota("my-property/use/ns-ws-quota");
        admin.namespaces().deleteNamespace("my-property/use/ns-ws-quota");
    }
}

Example 3 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaManager method handleExceededBacklogQuota.

/**
 * Handle exceeded backlog by using policies set in the zookeeper for given topic
 *
 * @param persistentTopic
 *            Topic on which backlog has been exceeded
 */
public void handleExceededBacklogQuota(PersistentTopic persistentTopic) {
    TopicName topicName = TopicName.get(persistentTopic.getName());
    String namespace = topicName.getNamespace();
    String policyPath = AdminResource.path(POLICIES, namespace);
    BacklogQuota quota = getBacklogQuota(namespace, policyPath);
    log.info("Backlog quota exceeded for topic [{}]. Applying [{}] policy", persistentTopic.getName(), quota.getPolicy());
    switch(quota.getPolicy()) {
        case consumer_backlog_eviction:
            dropBacklog(persistentTopic, quota);
            break;
        case producer_exception:
        case producer_request_hold:
            disconnectProducers(persistentTopic);
            break;
        default:
            break;
    }
}

Example 4 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaManagerTest method testConcurrentAckAndEviction.

@Test
public void testConcurrentAckAndEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic12";
    final String subName1 = "c12";
    final String subName2 = "c22";
    final int numMsgs = 20;
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer1 = client2.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    Consumer<byte[]> consumer2 = client2.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    Thread producerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs; i++) {
                    // only one consumer acknowledges the message
                    consumer1.acknowledge(consumer1.receive());
                    consumer2.receive();
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread.start();
    ConsumerThread.start();
    // test hangs without timeout since there is nothing to consume due to eviction
    counter.await(20, TimeUnit.SECONDS);
    assertTrue(!gotException.get());
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize <= 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
    client2.close();
}

Example 5 with BacklogQuota

use of org.apache.pulsar.common.policies.data.BacklogQuota in project incubator-pulsar by apache.

the class BacklogQuotaManagerTest method testNoEviction.

@Test
public void testNoEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic13";
    final String subName1 = "c13";
    final String subName2 = "c23";
    final int numMsgs = 10;
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    final Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    final PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Thread producerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                org.apache.pulsar.client.api.Producer<byte[]> producer = client2.newProducer().topic(topic1).create();
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs; i++) {
                    consumer1.acknowledge(consumer1.receive());
                    consumer2.acknowledge(consumer2.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread.start();
    ConsumerThread.start();
    counter.await();
    assertTrue(!gotException.get());
    client.close();
    client2.close();
}