use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentTopicTest method testPublishMessage.
@Test
public void testPublishMessage() throws Exception {
PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
/*
* MessageMetadata.Builder messageMetadata = MessageMetadata.newBuilder();
* messageMetadata.setPublishTime(System.currentTimeMillis()); messageMetadata.setProducerName("producer-name");
* messageMetadata.setSequenceId(1);
*/
ByteBuf payload = Unpooled.wrappedBuffer("content".getBytes());
final CountDownLatch latch = new CountDownLatch(1);
topic.publishMessage(payload, (exception, ledgerId, entryId) -> {
latch.countDown();
});
assertTrue(latch.await(1, TimeUnit.SECONDS));
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentTopicTest method testDispatcherMultiConsumerReadFailed.
@Test
public void testDispatcherMultiConsumerReadFailed() throws Exception {
PersistentTopic topic = spy(new PersistentTopic(successTopicName, ledgerMock, brokerService));
ManagedCursor cursor = mock(ManagedCursor.class);
when(cursor.getName()).thenReturn("cursor");
PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursor);
dispatcher.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
verify(topic, atLeast(1)).getBrokerService();
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentTopicE2ETest method testCompression.
@Test(dataProvider = "codec")
public void testCompression(CompressionType compressionType) throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic0" + compressionType;
// 1. producer connect
ProducerConfiguration producerConf = new ProducerConfiguration();
producerConf.setCompressionType(compressionType);
Producer producer = pulsarClient.createProducer(topicName, producerConf);
Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. producer publish messages
for (int i = 0; i < 10; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
for (int i = 0; i < 10; i++) {
Message msg = consumer.receive(5, TimeUnit.SECONDS);
assertNotNull(msg);
assertEquals(msg.getData(), ("my-message-" + i).getBytes());
}
// 3. producer disconnect
producer.close();
consumer.close();
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentTopicE2ETest method testActiveSubscriptionWithCache.
/**
* Validation: 1. validates active-cursor after active subscription 2. validate active-cursor with subscription 3.
* unconsumed messages should be present into cache 4. cache and active-cursor should be empty once subscription is
* closed
*
* @throws Exception
*/
@Test
public void testActiveSubscriptionWithCache() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/topic2";
final String subName = "sub2";
Message msg;
int recvQueueSize = 4;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Exclusive);
conf.setReceiverQueueSize(recvQueueSize);
// (1) Create subscription
Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
Producer producer = pulsarClient.createProducer(topicName);
// (2) Produce Messages
for (int i = 0; i < recvQueueSize / 2; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
msg = consumer.receive();
consumer.acknowledge(msg);
}
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
// (3) Get Entry cache
ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
cacheField.setAccessible(true);
EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);
/************* Validation on non-empty active-cursor **************/
// (4) Get ActiveCursor : which is list of active subscription
Iterable<ManagedCursor> activeCursors = ledger.getActiveCursors();
ManagedCursor curosr = activeCursors.iterator().next();
// (4.1) Validate: active Cursor must be non-empty
assertNotNull(curosr);
// (4.2) Validate: validate cursor name
assertEquals(subName, curosr.getName());
// (4.3) Validate: entryCache should have cached messages
assertTrue(entryCache.getSize() != 0);
/************* Validation on empty active-cursor **************/
// (5) Close consumer: which (1)removes activeConsumer and (2)clears the entry-cache
consumer.close();
Thread.sleep(1000);
// (5.1) Validate: active-consumer must be empty
assertFalse(ledger.getActiveCursors().iterator().hasNext());
// (5.2) Validate: Entry-cache must be cleared
assertTrue(entryCache.getSize() == 0);
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentTopicE2ETest method testConcurrentConsumerThreads.
// some race conditions needs to be handled
// disabling the test for now to not block commit jobs
@Test(enabled = false)
public void testConcurrentConsumerThreads() throws Exception {
// test concurrent consumer threads on same consumerId
final String topicName = "persistent://prop/use/ns-abc/topic3";
final String subName = "sub3";
final int recvQueueSize = 100;
final int numConsumersThreads = 10;
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setSubscriptionType(SubscriptionType.Exclusive);
conf.setReceiverQueueSize(recvQueueSize);
ExecutorService executor = Executors.newCachedThreadPool();
final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
for (int i = 0; i < numConsumersThreads; i++) {
executor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
barrier.await();
Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
for (int i = 0; i < recvQueueSize / numConsumersThreads; i++) {
Message msg = consumer.receive();
consumer.acknowledge(msg);
}
return null;
}
});
}
Producer producer = pulsarClient.createProducer(topicName);
for (int i = 0; i < recvQueueSize * numConsumersThreads; i++) {
String message = "my-message-" + i;
producer.send(message.getBytes());
}
barrier.await();
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
// 1. cumulatively all threads drain the backlog
assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
// 2. flow control works the same as single consumer single thread
Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
assertEquals(getAvailablePermits(subRef), recvQueueSize);
}
Aggregations