Examples with PersistentTopicStats com.yahoo.pulsar.common.policies.data.PersistentTopicStats used on opensource projects

Example 6 with PersistentTopicStats

use of com.yahoo.pulsar.common.policies.data.PersistentTopicStats in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testEvictionMulti.

@Test
public void testEvictionMulti() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(15 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic14";
    final String subName1 = "c14";
    final String subName2 = "c24";
    final int numMsgs = 10;
    final CyclicBarrier barrier = new CyclicBarrier(4);
    final CountDownLatch counter = new CountDownLatch(4);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final Consumer consumer1 = client.subscribe(topic1, subName1);
    final Consumer consumer2 = client.subscribe(topic1, subName2);
    final PulsarClient client3 = PulsarClient.create(adminUrl.toString(), clientConf);
    final PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
    Thread producerThread1 = new Thread() {

        public void run() {
            try {
                barrier.await();
                com.yahoo.pulsar.client.api.Producer producer = client2.createProducer(topic1);
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread producerThread2 = new Thread() {

        public void run() {
            try {
                barrier.await();
                com.yahoo.pulsar.client.api.Producer producer = client3.createProducer(topic1);
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread1 = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs * 2; i++) {
                    consumer1.acknowledge(consumer1.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread2 = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs * 2; i++) {
                    consumer2.acknowledge(consumer2.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread1.start();
    producerThread2.start();
    ConsumerThread1.start();
    ConsumerThread2.start();
    counter.await(20, TimeUnit.SECONDS);
    assertTrue(!gotException.get());
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize <= 15 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
    client2.close();
    client3.close();
}

Example 7 with PersistentTopicStats

use of com.yahoo.pulsar.common.policies.data.PersistentTopicStats in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testConsumerBacklogEviction.

@Test
public void testConsumerBacklogEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/ns-quota/topic1";
    final String subName1 = "c1";
    final String subName2 = "c2";
    final int numMsgs = 20;
    Consumer consumer1 = client.subscribe(topic1, subName1);
    Consumer consumer2 = client.subscribe(topic1, subName2);
    com.yahoo.pulsar.client.api.Producer producer = client.createProducer(topic1);
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        consumer1.receive();
        consumer2.receive();
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize < 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}

Example 8 with PersistentTopicStats

use of com.yahoo.pulsar.common.policies.data.PersistentTopicStats in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testConsumerBacklogEvictionWithAck.

@Test
public void testConsumerBacklogEvictionWithAck() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    PulsarClient client = PulsarClient.create(adminUrl.toString());
    final String topic1 = "persistent://prop/usc/ns-quota/topic11";
    final String subName1 = "c11";
    final String subName2 = "c21";
    final int numMsgs = 20;
    Consumer consumer1 = client.subscribe(topic1, subName1);
    Consumer consumer2 = client.subscribe(topic1, subName2);
    com.yahoo.pulsar.client.api.Producer producer = client.createProducer(topic1);
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        // only one consumer acknowledges the message
        consumer1.acknowledge(consumer1.receive());
        consumer2.receive();
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize <= 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}

Example 9 with PersistentTopicStats

use of com.yahoo.pulsar.common.policies.data.PersistentTopicStats in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testProducerExceptionAndThenUnblock.

@Test
public void testProducerExceptionAndThenUnblock() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_exception));
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/quotahold/exceptandunblock";
    final String subName1 = "c1except";
    boolean gotException = false;
    Consumer consumer = client.subscribe(topic1, subName1);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
    byte[] content = new byte[1024];
    Producer producer = client.createProducer(topic1, producerConfiguration);
    for (int i = 0; i < 10; i++) {
        producer.send(content);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    try {
        // try to send over backlog quota and make sure it fails
        producer.send(content);
        producer.send(content);
        Assert.fail("backlog quota did not exceed");
    } catch (PulsarClientException ce) {
        Assert.assertTrue(ce instanceof PulsarClientException.ProducerBlockedQuotaExceededException || ce instanceof PulsarClientException.TimeoutException, ce.getMessage());
        gotException = true;
    }
    Assert.assertTrue(gotException, "backlog exceeded exception did not occur");
    // now remove backlog and ensure that producer is unblockedrolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    int backlog = (int) stats.subscriptions.get(subName1).msgBacklog;
    for (int i = 0; i < backlog; i++) {
        Message msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    // publish should work now
    Exception sendException = null;
    gotException = false;
    try {
        for (int i = 0; i < 5; i++) {
            producer.send(content);
        }
    } catch (Exception e) {
        gotException = true;
        sendException = e;
    }
    Assert.assertFalse(gotException, "unable to publish due to " + sendException);
    client.close();
}

Example 10 with PersistentTopicStats

use of com.yahoo.pulsar.common.policies.data.PersistentTopicStats in project pulsar by yahoo.

the class PersistentTopics method getPartitionedStats.

@GET
@Path("{property}/{cluster}/{namespace}/{destination}/partitioned-stats")
@ApiOperation(value = "Get the stats for the partitioned topic.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic does not exist") })
public PartitionedTopicStats getPartitionedStats(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    destination = decode(destination);
    DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
    if (partitionMetadata.partitions == 0) {
        throw new RestException(Status.NOT_FOUND, "Partitioned Topic not found");
    }
    PartitionedTopicStats stats = new PartitionedTopicStats(partitionMetadata);
    try {
        for (int i = 0; i < partitionMetadata.partitions; i++) {
            PersistentTopicStats partitionStats = pulsar().getAdminClient().persistentTopics().getStats(dn.getPartition(i).toString());
            stats.add(partitionStats);
            stats.partitions.put(dn.getPartition(i).toString(), partitionStats);
        }
    } catch (Exception e) {
        throw new RestException(e);
    }
    return stats;
}