use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class PersistentTopicE2ETest method testDeleteTopics.
@Test
public void testDeleteTopics() throws Exception {
BrokerService brokerService = pulsar.getBrokerService();
// 1. producers connect
Producer producer1 = pulsarClient.createProducer("persistent://prop/use/ns-abc/topic-1");
Producer producer2 = pulsarClient.createProducer("persistent://prop/use/ns-abc/topic-2");
brokerService.updateRates();
Map<String, NamespaceBundleStats> bundleStatsMap = brokerService.getBundleStats();
assertEquals(bundleStatsMap.size(), 1);
NamespaceBundleStats bundleStats = bundleStatsMap.get("prop/use/ns-abc/0x00000000_0xffffffff");
assertNotNull(bundleStats);
producer1.close();
admin.persistentTopics().delete("persistent://prop/use/ns-abc/topic-1");
brokerService.updateRates();
bundleStatsMap = brokerService.getBundleStats();
assertEquals(bundleStatsMap.size(), 1);
bundleStats = bundleStatsMap.get("prop/use/ns-abc/0x00000000_0xffffffff");
assertNotNull(bundleStats);
// // Delete 2nd topic as well
// producer2.close();
// admin.persistentTopics().delete("persistent://prop/use/ns-abc/topic-2");
//
// brokerService.updateRates();
//
// bundleStatsMap = brokerService.getBundleStats();
// assertEquals(bundleStatsMap.size(), 0);
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class PersistentTopicE2ETest method testCompression.
@Test(dataProvider = "codec")
public void testCompression(CompressionType compressionType) throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic0" + compressionType;
// 1. producer connect
ProducerConfiguration producerConf = new ProducerConfiguration();
producerConf.setCompressionType(compressionType);
Producer producer = pulsarClient.createProducer(topicName, producerConf);
Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. producer publish messages
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
for (int i = 0; i < 10; i++) {
Message msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
assertEquals(msg.getData(), ("my-message-" + i).getBytes());
}
// 3. producer disconnect
producer.close();
consumer.close();
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class PersistentTopicE2ETest method testActiveSubscriptionWithCache.
/**
* Validation: 1. validates active-cursor after active subscription 2. validate active-cursor with subscription 3.
* unconsumed messages should be present into cache 4. cache and active-cursor should be empty once subscription is
* closed
*
* @throws Exception
*/
@Test
public void testActiveSubscriptionWithCache() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic2";
final String subName = "sub2";
Message msg;
int recvQueueSize = 4;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Exclusive);
conf.setReceiverQueueSize(recvQueueSize);
// (1) Create subscription
Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
Producer producer = pulsarClient.createProducer(topicName);
// (2) Produce Messages
for (int i = 0; i < recvQueueSize / 2; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
msg = consumer.receive();
consumer.acknowledge(msg);
}
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
// (3) Get Entry cache
ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
cacheField.setAccessible(true);
EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);
/************* Validation on non-empty active-cursor **************/
// (4) Get ActiveCursor : which is list of active subscription
Iterable<ManagedCursor> activeCursors = ledger.getActiveCursors();
ManagedCursor curosr = activeCursors.iterator().next();
// (4.1) Validate: active Cursor must be non-empty
assertNotNull(curosr);
// (4.2) Validate: validate cursor name
assertEquals(subName, curosr.getName());
// (4.3) Validate: entryCache should have cached messages
assertTrue(entryCache.getSize() != 0);
/************* Validation on empty active-cursor **************/
// (5) Close consumer: which (1)removes activeConsumer and (2)clears the entry-cache
consumer.close();
Thread.sleep(1000);
// (5.1) Validate: active-consumer must be empty
assertFalse(ledger.getActiveCursors().iterator().hasNext());
// (5.2) Validate: Entry-cache must be cleared
assertTrue(entryCache.getSize() == 0);
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class PersistentTopicE2ETest method testConcurrentConsumerThreads.
// some race conditions needs to be handled
// disabling the test for now to not block commit jobs
@Test(enabled = false)
public void testConcurrentConsumerThreads() throws Exception {
// test concurrent consumer threads on same consumerId
final String topicName = "persistent://prop/use/ns-abc/topic3";
final String subName = "sub3";
final int recvQueueSize = 100;
final int numConsumersThreads = 10;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Exclusive);
conf.setReceiverQueueSize(recvQueueSize);
ExecutorService executor = Executors.newCachedThreadPool();
final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
for (int i = 0; i < numConsumersThreads; i++) {
executor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
barrier.await();
Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
for (int i = 0; i < recvQueueSize / numConsumersThreads; i++) {
Message msg = consumer.receive();
consumer.acknowledge(msg);
}
return null;
}
});
}
Producer producer = pulsarClient.createProducer(topicName);
for (int i = 0; i < recvQueueSize * numConsumersThreads; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
barrier.await();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
// 1. cumulatively all threads drain the backlog
assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
// 2. flow control works the same as single consumer single thread
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
assertEquals(getAvailablePermits(subRef), recvQueueSize);
}
use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.
the class PersistentTopicE2ETest method testBrokerTopicStats.
@Test
public void testBrokerTopicStats() throws Exception {
BrokerService brokerService = this.pulsar.getBrokerService();
Field field = BrokerService.class.getDeclaredField("statsUpdater");
field.setAccessible(true);
ScheduledExecutorService statsUpdater = (ScheduledExecutorService) field.get(brokerService);
// disable statsUpdate to calculate rates explicitly
statsUpdater.shutdown();
final String namespace = "prop/use/ns-abc";
ProducerConfiguration producerConf = new ProducerConfiguration();
Producer producer = pulsarClient.createProducer("persistent://" + namespace + "/topic0", producerConf);
// 1. producer publish messages
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
Metrics metric = null;
// sleep 1 sec to caclulate metrics per second
Thread.sleep(1000);
brokerService.updateRates();
List<Metrics> metrics = brokerService.getDestinationMetrics();
for (int i = 0; i < metrics.size(); i++) {
if (metrics.get(i).getDimension("namespace").equalsIgnoreCase(namespace)) {
metric = metrics.get(i);
break;
}
}
assertNotNull(metric);
double msgInRate = (double) metrics.get(0).getMetrics().get("brk_in_rate");
// rate should be calculated and no must be > 0 as we have produced 10 msgs so far
assertTrue(msgInRate > 0);
}
Aggregations