use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.
the class PulsarAdminToolTest method namespaces.
@Test
void namespaces() throws Exception {
PulsarAdmin admin = Mockito.mock(PulsarAdmin.class);
Namespaces mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
Lookup mockLookup = mock(Lookup.class);
when(admin.lookups()).thenReturn(mockLookup);
CmdNamespaces namespaces = new CmdNamespaces(admin);
namespaces.run(split("list myprop"));
verify(mockNamespaces).getNamespaces("myprop");
namespaces.run(split("list-cluster myprop/clust"));
verify(mockNamespaces).getNamespaces("myprop", "clust");
namespaces.run(split("destinations myprop/clust/ns1"));
verify(mockNamespaces).getDestinations("myprop/clust/ns1");
namespaces.run(split("policies myprop/clust/ns1"));
verify(mockNamespaces).getPolicies("myprop/clust/ns1");
namespaces.run(split("create myprop/clust/ns1"));
verify(mockNamespaces).createNamespace("myprop/clust/ns1");
namespaces.run(split("delete myprop/clust/ns1"));
verify(mockNamespaces).deleteNamespace("myprop/clust/ns1");
namespaces.run(split("permissions myprop/clust/ns1"));
verify(mockNamespaces).getPermissions("myprop/clust/ns1");
namespaces.run(split("grant-permission myprop/clust/ns1 --role role1 --actions produce,consume"));
verify(mockNamespaces).grantPermissionOnNamespace("myprop/clust/ns1", "role1", EnumSet.of(AuthAction.produce, AuthAction.consume));
namespaces.run(split("revoke-permission myprop/clust/ns1 --role role1"));
verify(mockNamespaces).revokePermissionsOnNamespace("myprop/clust/ns1", "role1");
namespaces.run(split("set-clusters myprop/clust/ns1 -c use,usw,usc"));
verify(mockNamespaces).setNamespaceReplicationClusters("myprop/clust/ns1", Lists.newArrayList("use", "usw", "usc"));
namespaces.run(split("get-clusters myprop/clust/ns1"));
verify(mockNamespaces).getNamespaceReplicationClusters("myprop/clust/ns1");
namespaces.run(split("unload myprop/clust/ns1"));
verify(mockNamespaces).unload("myprop/clust/ns1");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("unload myprop/clust/ns1 -b 0x80000000_0xffffffff"));
verify(mockNamespaces).unloadNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff");
namespaces.run(split("split-bundle myprop/clust/ns1 -b 0x00000000_0xffffffff"));
verify(mockNamespaces).splitNamespaceBundle("myprop/clust/ns1", "0x00000000_0xffffffff");
namespaces.run(split("get-backlog-quotas myprop/clust/ns1"));
verify(mockNamespaces).getBacklogQuotaMap("myprop/clust/ns1");
namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_request_hold -l 10"));
verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10, RetentionPolicy.producer_request_hold));
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10K"));
verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024, RetentionPolicy.producer_exception));
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10M"));
verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10 * 1024 * 1024, RetentionPolicy.producer_exception));
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("set-backlog-quota myprop/clust/ns1 -p producer_exception -l 10G"));
verify(mockNamespaces).setBacklogQuota("myprop/clust/ns1", new BacklogQuota(10l * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
namespaces.run(split("set-persistence myprop/clust/ns1 -e 2 -w 1 -a 1 -r 100.0"));
verify(mockNamespaces).setPersistence("myprop/clust/ns1", new PersistencePolicies(2, 1, 1, 100.0d));
namespaces.run(split("get-persistence myprop/clust/ns1"));
verify(mockNamespaces).getPersistence("myprop/clust/ns1");
namespaces.run(split("set-message-ttl myprop/clust/ns1 -ttl 300"));
verify(mockNamespaces).setNamespaceMessageTTL("myprop/clust/ns1", 300);
namespaces.run(split("get-message-ttl myprop/clust/ns1"));
verify(mockNamespaces).getNamespaceMessageTTL("myprop/clust/ns1");
namespaces.run(split("set-retention myprop/clust/ns1 -t 1h -s 1M"));
verify(mockNamespaces).setRetention("myprop/clust/ns1", new RetentionPolicies(60, 1));
namespaces.run(split("get-retention myprop/clust/ns1"));
verify(mockNamespaces).getRetention("myprop/clust/ns1");
namespaces.run(split("clear-backlog myprop/clust/ns1 -force"));
verify(mockNamespaces).clearNamespaceBacklog("myprop/clust/ns1");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff myprop/clust/ns1 -force"));
verify(mockNamespaces).clearNamespaceBundleBacklog("myprop/clust/ns1", "0x80000000_0xffffffff");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("clear-backlog -s my-sub myprop/clust/ns1 -force"));
verify(mockNamespaces).clearNamespaceBacklogForSubscription("myprop/clust/ns1", "my-sub");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("clear-backlog -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1 -force"));
verify(mockNamespaces).clearNamespaceBundleBacklogForSubscription("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
namespaces.run(split("unsubscribe -s my-sub myprop/clust/ns1"));
verify(mockNamespaces).unsubscribeNamespace("myprop/clust/ns1", "my-sub");
mockNamespaces = mock(Namespaces.class);
when(admin.namespaces()).thenReturn(mockNamespaces);
namespaces = new CmdNamespaces(admin);
namespaces.run(split("unsubscribe -b 0x80000000_0xffffffff -s my-sub myprop/clust/ns1"));
verify(mockNamespaces).unsubscribeNamespaceBundle("myprop/clust/ns1", "0x80000000_0xffffffff", "my-sub");
}
use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.
the class PersistentTopic method getBacklogQuota.
/**
*
* @return Backlog quota for topic
*/
@Override
public BacklogQuota getBacklogQuota() {
DestinationName destination = DestinationName.get(this.getName());
String namespace = destination.getNamespace();
String policyPath = AdminResource.path("policies", namespace);
BacklogQuota backlogQuota = brokerService.getBacklogQuotaManager().getBacklogQuota(namespace, policyPath);
return backlogQuota;
}
use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.
the class AdminApiTest method backlogQuotas.
@Test
public void backlogQuotas() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop-xyz/use/ns1"), Maps.newTreeMap());
Map<BacklogQuotaType, BacklogQuota> quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/use/ns1");
assertEquals(quotaMap.size(), 0);
assertEquals(quotaMap.get(BacklogQuotaType.destination_storage), null);
admin.namespaces().setBacklogQuota("prop-xyz/use/ns1", new BacklogQuota(1 * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/use/ns1");
assertEquals(quotaMap.size(), 1);
assertEquals(quotaMap.get(BacklogQuotaType.destination_storage), new BacklogQuota(1 * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
admin.namespaces().removeBacklogQuota("prop-xyz/use/ns1");
quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/use/ns1");
assertEquals(quotaMap.size(), 0);
assertEquals(quotaMap.get(BacklogQuotaType.destination_storage), null);
}
use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.
the class BacklogQuotaManagerTest method testConcurrentAckAndEviction.
@Test
public void testConcurrentAckAndEviction() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
final String topic1 = "persistent://prop/usc/ns-quota/topic12";
final String subName1 = "c12";
final String subName2 = "c22";
final int numMsgs = 20;
final CyclicBarrier barrier = new CyclicBarrier(2);
final CountDownLatch counter = new CountDownLatch(2);
final AtomicBoolean gotException = new AtomicBoolean(false);
ClientConfiguration clientConf = new ClientConfiguration();
clientConf.setStatsInterval(0, TimeUnit.SECONDS);
PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
Consumer consumer1 = client2.subscribe(topic1, subName1);
Consumer consumer2 = client2.subscribe(topic1, subName2);
Thread producerThread = new Thread() {
public void run() {
try {
barrier.await();
final com.yahoo.pulsar.client.api.Producer producer = client.createProducer(topic1);
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
}
producer.close();
} catch (Exception e) {
gotException.set(true);
} finally {
counter.countDown();
}
}
};
Thread ConsumerThread = new Thread() {
public void run() {
try {
barrier.await();
for (int i = 0; i < numMsgs; i++) {
// only one consumer acknowledges the message
consumer1.acknowledge(consumer1.receive());
consumer2.receive();
}
} catch (Exception e) {
gotException.set(true);
} finally {
counter.countDown();
}
}
};
producerThread.start();
ConsumerThread.start();
// test hangs without timeout since there is nothing to consume due to eviction
counter.await(20, TimeUnit.SECONDS);
assertTrue(!gotException.get());
Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
rolloverStats();
PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
Assert.assertTrue(stats.storageSize <= 10 * 1024, "Storage size is [" + stats.storageSize + "]");
client.close();
client2.close();
}
use of com.yahoo.pulsar.common.policies.data.BacklogQuota in project pulsar by yahoo.
the class BacklogQuotaManagerTest method testEvictionMulti.
@Test
public void testEvictionMulti() throws Exception {
assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(15 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
final String topic1 = "persistent://prop/usc/ns-quota/topic14";
final String subName1 = "c14";
final String subName2 = "c24";
final int numMsgs = 10;
final CyclicBarrier barrier = new CyclicBarrier(4);
final CountDownLatch counter = new CountDownLatch(4);
final AtomicBoolean gotException = new AtomicBoolean(false);
final ClientConfiguration clientConf = new ClientConfiguration();
clientConf.setStatsInterval(0, TimeUnit.SECONDS);
final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
final Consumer consumer1 = client.subscribe(topic1, subName1);
final Consumer consumer2 = client.subscribe(topic1, subName2);
final PulsarClient client3 = PulsarClient.create(adminUrl.toString(), clientConf);
final PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
Thread producerThread1 = new Thread() {
public void run() {
try {
barrier.await();
com.yahoo.pulsar.client.api.Producer producer = client2.createProducer(topic1);
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
}
producer.close();
} catch (Exception e) {
gotException.set(true);
} finally {
counter.countDown();
}
}
};
Thread producerThread2 = new Thread() {
public void run() {
try {
barrier.await();
com.yahoo.pulsar.client.api.Producer producer = client3.createProducer(topic1);
byte[] content = new byte[1024];
for (int i = 0; i < numMsgs; i++) {
producer.send(content);
}
producer.close();
} catch (Exception e) {
gotException.set(true);
} finally {
counter.countDown();
}
}
};
Thread ConsumerThread1 = new Thread() {
public void run() {
try {
barrier.await();
for (int i = 0; i < numMsgs * 2; i++) {
consumer1.acknowledge(consumer1.receive());
}
} catch (Exception e) {
gotException.set(true);
} finally {
counter.countDown();
}
}
};
Thread ConsumerThread2 = new Thread() {
public void run() {
try {
barrier.await();
for (int i = 0; i < numMsgs * 2; i++) {
consumer2.acknowledge(consumer2.receive());
}
} catch (Exception e) {
gotException.set(true);
} finally {
counter.countDown();
}
}
};
producerThread1.start();
producerThread2.start();
ConsumerThread1.start();
ConsumerThread2.start();
counter.await(20, TimeUnit.SECONDS);
assertTrue(!gotException.get());
Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
rolloverStats();
PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
Assert.assertTrue(stats.storageSize <= 15 * 1024, "Storage size is [" + stats.storageSize + "]");
client.close();
client2.close();
client3.close();
}
Aggregations